Compare commits

..
Author SHA1 Message Date
Henry ParkandClaude Opus 4.6 650c914029 fix: re-apply correct download URLs for telegram-mtproto and slack-tool
Re-apply the URL corrections after local revert.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
2026-03-02 12:01:47 -08:00
389 changed files with 5797 additions and 60171 deletions
+2 -34
View File
@@ -5,19 +5,6 @@ DATABASE_POOL_SIZE=10
# LLM Provider
# LLM_BACKEND=nearai # default
# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil
# LLM_REQUEST_TIMEOUT_SECS=120 # Increase for local LLMs (Ollama, vLLM, LM Studio)
# === Anthropic Direct ===
# Two auth modes:
# 1. API key: Set ANTHROPIC_API_KEY (from console.anthropic.com/settings/keys)
# 2. OAuth token: Set ANTHROPIC_OAUTH_TOKEN (from `claude login`)
# OAuth tokens use Authorization: Bearer instead of x-api-key header.
# ANTHROPIC_API_KEY=sk-ant-...
# ANTHROPIC_OAUTH_TOKEN=sk-ant-oat01-... # from `claude login` credentials
# ANTHROPIC_MODEL=claude-sonnet-4-20250514
# === OpenAI Direct ===
# OPENAI_API_KEY=sk-...
# === NEAR AI (Chat Completions API) ===
# Two auth modes:
@@ -70,17 +57,6 @@ NEARAI_AUTH_URL=https://private.near.ai
# LLM_BASE_URL=https://api.fireworks.ai/inference/v1
# LLM_API_KEY=fw_...
# === Anthropic Direct ===
# LLM_BACKEND=anthropic
# ANTHROPIC_MODEL=claude-sonnet-4-6
# ANTHROPIC_API_KEY=sk-ant-...
# ANTHROPIC_BASE_URL=https://api.anthropic.com # default
# Prompt cache retention — controls Anthropic server-side prompt caching:
# none = disabled (no cache_control injected)
# short = 5-minute TTL, 1.25× (125%) write surcharge (default)
# long = 1-hour TTL, 2.0× (200%) write surcharge
# ANTHROPIC_CACHE_RETENTION=short
# For full provider setup guide see docs/LLM_PROVIDERS.md
# Channel Configuration
@@ -132,20 +108,12 @@ HEARTBEAT_NOTIFY_USER=default
# Memory hygiene settings (automatic cleanup of stale workspace documents)
# Runs on each heartbeat tick; identity files (IDENTITY.md, SOUL.md) are never deleted
# MEMORY_HYGIENE_ENABLED=true
# MEMORY_HYGIENE_DAILY_RETENTION_DAYS=30 # delete daily/ docs older than this many days
# MEMORY_HYGIENE_CONVERSATION_RETENTION_DAYS=7 # delete conversations/ docs older than this many days
# MEMORY_HYGIENE_CADENCE_HOURS=12 # minimum hours between cleanup passes
# MEMORY_HYGIENE_RETENTION_DAYS=30 # delete daily/ docs older than this many days
# MEMORY_HYGIENE_CADENCE_HOURS=12 # minimum hours between cleanup passes
# Safety settings
SAFETY_MAX_OUTPUT_LENGTH=100000
SAFETY_INJECTION_CHECK_ENABLED=true
# Restart Feature (Docker containers only)
# Set IRONCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature.
# Without this, the restart tool and /restart command will be disabled.
# IRONCLAW_IN_DOCKER=false
# IRONCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30)
# IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits
# Logging
RUST_LOG=ironclaw=debug,tower_http=debug
-1
View File
@@ -1 +0,0 @@
../scripts/commit-msg-regression.sh
-24
View File
@@ -1,24 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# Pre-commit hook: run version bump checks when WIT or extension sources change.
# Install: git config core.hooksPath .githooks
# Only run the check if relevant files are staged
STAGED=$(git diff --cached --name-only)
NEEDS_CHECK=false
if echo "$STAGED" | grep -qE '^wit/|^channels-src/|^tools-src/'; then
NEEDS_CHECK=true
fi
if $NEEDS_CHECK; then
echo "pre-commit: checking version bumps..."
if ! ./scripts/check-version-bumps.sh; then
echo ""
echo "Commit blocked: version bump check failed."
echo "Bump versions in the relevant registry JSON and/or WIT package declaration."
echo "To bypass: git commit --no-verify"
exit 1
fi
fi
-3
View File
@@ -62,9 +62,6 @@ create "scope: ci" "546E7A" "CI/CD workflows"
create "scope: docs" "78909C" "Documentation"
create "scope: dependencies" "90A4AE" "Dependency updates"
echo "==> Creating workflow labels..."
create "skip-regression-check" "9E9E9E" "Acknowledged: fix without regression test"
echo "==> Creating contributor labels..."
create "contributor: new" "FFF9C4" "First-time contributor"
create "contributor: regular" "FFE082" "2-5 merged PRs"
-99
View File
@@ -1,99 +0,0 @@
name: Claude Code Review
on:
pull_request:
types: [opened, labeled]
permissions:
contents: read
pull-requests: write
issues: write
id-token: write
concurrency:
group: claude-review-${{ github.event.pull_request.number || github.run_id }}
cancel-in-progress: true
jobs:
review:
name: Claude Code Review
if: contains(github.event.pull_request.labels.*.name, 'staging-promotion')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Run Claude Code review
uses: anthropics/claude-code-action@v1
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
claude_args: "--max-turns 50 --model claude-haiku-4-5-20251001 --allowedTools 'Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*),Bash(gh issue view:*),Bash(gh issue list:*),Bash(gh search:*),Bash(git blame:*),Bash(git log:*),Bash(git diff:*)'"
prompt: |
Code review this pull request. Follow these steps precisely:
1. Use a Haiku agent to find relevant CLAUDE.md files: the root CLAUDE.md
and any CLAUDE.md files in directories whose files this PR modifies.
2. Use a Haiku agent to summarize the PR change (use `gh pr diff`).
3. Launch 4 parallel agents to review the change independently. Each agent should
read the PR diff with `gh pr diff` and the full source files for changed
code, then return a list of issues found:
Agent 1 — Security & Safety
Check for: command injection, path traversal, SSRF, XSS, auth bypass,
secrets in logs, .unwrap()/.expect() in production code (not tests),
race conditions, TOCTOU, unsafe blocks, panics in async, unbounded allocations.
Agent 2 — Architecture & Patterns
Check for: extensible design (traits/enums over nested conditionals),
clean abstractions, proper error types (thiserror), CLAUDE.md compliance,
type-driven design over stringly-typed code, DRY violations.
Agent 3 — Bug Scan
Shallow diff-only scan for obvious bugs: logic errors, off-by-one,
missing error handling, division by zero, incorrect return values.
Ignore nitpicks and likely false positives. Do NOT read extra context
beyond the diff — focus only on the changes.
Agent 4 — Performance & Production
Check for: blocking in async, N+1 queries, unbounded loops, missing
timeouts, resource leaks (file handles, connections), large allocations
in hot paths.
4. For each issue found, launch a parallel Haiku agent to:
a. Assign a severity:
- CRITICAL: security vulns, panics in prod (.unwrap/.expect), data exfiltration, race conditions
- HIGH: logic bugs, missing error handling, breaking API/schema changes
- MEDIUM: missing tests, unnecessary complexity, performance issues
- LOW: documentation gaps, naming suggestions
b. Score confidence 0-100 (give this rubric verbatim):
0: False positive, doesn't stand up to scrutiny, or pre-existing issue.
25: Might be real, but may be false positive. Stylistic issues not in CLAUDE.md.
50: Real issue but nitpick or rare in practice. Not very important.
75: Verified real issue, will be hit in practice. Directly impacts functionality
or explicitly mentioned in CLAUDE.md.
100: Certain, confirmed, will happen frequently. Evidence directly confirms.
5. Post a single comment on the PR using `gh pr comment` with this format.
If no issues were found, post "No issues found." instead:
### Code review
Found N issues:
1. [SEVERITY:CONFIDENCE] <brief description>
<permalink to file:line using full SHA, eg https://github.com/owner/repo/blob/abc123def/src/file.rs#L10-L15>
Example: [CRITICAL:92] `.unwrap()` can panic in production when config is missing
You MUST use the full git SHA in links (not HEAD or branch name).
Provide 1 line of context before and after each linked range.
Notes:
- Use `gh` for all GitHub interactions, not web fetch
- Do NOT check build signal or attempt to build/test the code
- Ignore pre-existing issues not introduced by this PR
- Ignore issues a linter/compiler would catch (formatting, imports, types)
+4 -28
View File
@@ -12,6 +12,7 @@ jobs:
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
components: rustfmt
- name: Check formatting
run: cargo fmt --all -- --check
@@ -35,6 +36,7 @@ jobs:
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
components: clippy
- uses: Swatinem/rust-cache@v2
with:
@@ -42,41 +44,15 @@ jobs:
- name: Check lints
run: cargo clippy --all --benches --tests --examples ${{ matrix.flags }} -- -D warnings
clippy-windows:
name: Clippy Windows (${{ matrix.name }})
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- name: all-features
flags: "--all-features"
- name: default
flags: ""
- name: libsql-only
flags: "--no-default-features --features libsql"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- uses: Swatinem/rust-cache@v2
with:
key: clippy-windows-${{ matrix.name }}
- name: Check lints
run: cargo clippy --all --benches --tests --examples ${{ matrix.flags }} -- -D warnings
# Roll-up job for branch protection
code-style:
name: Code Style (fmt + clippy)
runs-on: ubuntu-latest
if: always()
needs: [format, clippy, clippy-windows]
needs: [format, clippy]
steps:
- run: |
if [[ "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" || "${{ needs.clippy-windows.result }}" != "success" ]]; then
if [[ "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" ]]; then
echo "One or more jobs failed"
exit 1
fi
-226
View File
@@ -1,226 +0,0 @@
# Code Coverage Workflow
#
# This workflow runs test coverage analysis and uploads reports to Codecov.
# Coverage reports help identify untested code paths and maintain code quality.
#
# What it does:
# - Runs unit and integration tests with coverage instrumentation
# - Runs E2E tests with coverage instrumentation
# - Uploads coverage reports to Codecov (https://codecov.io/gh/nearai/ironclaw)
#
# Viewing coverage reports:
# - PRs automatically get coverage comments showing changes in coverage
# - Visit https://codecov.io/gh/nearai/ironclaw for detailed coverage reports
# - Coverage reports are generated for three configurations:
# 1. all-features: Full feature set
# 2. default: Default features
# 3. libsql-only: Minimal libSQL-only configuration
# - E2E coverage tracks end-to-end test coverage separately
#
# Coverage files:
# - Unit/integration: lcov.info (uploaded to Codecov with "unit" flag)
# - E2E: e2e-coverage.info (uploaded to Codecov with "e2e" flag)
#
# Requirements:
# - Uses cargo-llvm-cov for coverage instrumentation
# - Requires PostgreSQL for integration tests (pgvector/pgvector:pg16)
# - E2E tests require Python 3.12 and Playwright
name: Code Coverage
on:
push:
branches: [main]
permissions:
id-token: write
contents: read
jobs:
coverage:
name: Coverage (${{ matrix.name }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- name: all-features
flags: "--all-features"
has_postgres: true
- name: default
flags: ""
has_postgres: true
- name: libsql-only
flags: "--no-default-features --features libsql"
has_postgres: false
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: ironclaw_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
targets: wasm32-wasip2
- uses: Swatinem/rust-cache@v2
with:
key: coverage-${{ matrix.name }}
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Install cargo-component
run: |
if ! command -v cargo-component >/dev/null 2>&1; then
cargo install cargo-component --locked
fi
- name: Build WASM channels (for integration tests)
run: ./scripts/build-wasm-extensions.sh --channels
- name: Run database migrations
if: matrix.has_postgres
run: |
set -euo pipefail
readarray -t migration_files < <(printf '%s\n' migrations/V*.sql | sort -V)
for f in "${migration_files[@]}"; do
echo "Applying $f..."
psql -v ON_ERROR_STOP=1 -f "$f"
done
env:
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: ironclaw_test
- name: Set DATABASE_URL for postgres configs
if: matrix.has_postgres
run: echo "DATABASE_URL=postgres://postgres:postgres@localhost/ironclaw_test" >> "$GITHUB_ENV"
- name: Generate coverage
run: cargo llvm-cov ${{ matrix.flags }} --workspace --lcov --output-path lcov.info
- name: Upload to Codecov
uses: codecov/codecov-action@v5
with:
files: lcov.info
flags: ${{ matrix.name }}
disable_search: true
use_oidc: true
fail_ci_if_error: true
e2e-coverage:
name: E2E Coverage
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
targets: wasm32-wasip2
- uses: Swatinem/rust-cache@v2
with:
key: e2e-coverage
- name: Install cargo-llvm-cov
uses: taiki-e/install-action@cargo-llvm-cov
- name: Install cargo-component
run: |
if ! command -v cargo-component >/dev/null 2>&1; then
cargo install cargo-component --locked
fi
- name: Build WASM channels
run: ./scripts/build-wasm-extensions.sh --channels
- name: Set up coverage instrumentation
run: |
# show-env outputs shell-quoted values (KEY='value') but GITHUB_ENV
# expects unquoted KEY=value. Strip only the wrapping single quotes
# from KEY='value' lines without altering any internal characters.
cargo llvm-cov show-env | sed -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'$/\1=\2/" >> "$GITHUB_ENV"
- name: Clean coverage workspace
run: cargo llvm-cov clean --workspace
- name: Build instrumented binary
run: cargo build --no-default-features --features libsql
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install E2E dependencies
run: |
cd tests/e2e
pip install -e .
playwright install --with-deps chromium
- name: Run E2E tests
run: |
pytest tests/e2e/ -v -x --timeout=120
env:
RUST_LOG: ironclaw=info
RUST_BACKTRACE: "1"
- name: Verify profraw files exist
if: always()
run: |
echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}"
echo "CARGO_LLVM_COV_TARGET_DIR=${CARGO_LLVM_COV_TARGET_DIR}"
profraw_count=$(find target/ -name '*.profraw' 2>/dev/null | wc -l)
echo "Found ${profraw_count} .profraw files under target/"
find target/ -name '*.profraw' 2>/dev/null || true
if [ "$profraw_count" -eq 0 ]; then
echo "::warning::No .profraw files found — coverage report will fail"
fi
- name: Generate coverage report
if: always()
run: cargo llvm-cov report --lcov --output-path e2e-coverage.info
- name: Upload to Codecov
if: always()
uses: codecov/codecov-action@v5
with:
files: e2e-coverage.info
flags: e2e
disable_search: true
use_oidc: true
fail_ci_if_error: true
- name: Upload screenshots on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-screenshots
path: tests/e2e/screenshots/
if-no-files-found: ignore
coverage-gate:
name: Coverage
runs-on: ubuntu-latest
if: always()
needs: [coverage, e2e-coverage]
steps:
- run: |
if [[ "${{ needs.coverage.result }}" != "success" || "${{ needs.e2e-coverage.result }}" != "success" ]]; then
echo "One or more coverage jobs failed"
exit 1
fi
+6 -56
View File
@@ -1,6 +1,5 @@
name: E2E Tests
on:
workflow_call:
schedule:
- cron: "0 6 * * 1" # Weekly Monday 6 AM UTC
workflow_dispatch:
@@ -10,9 +9,8 @@ on:
- "tests/e2e/**"
jobs:
# ── Step 1: compile once ──────────────────────────────────────────────────
build:
name: Build ironclaw (libsql)
e2e:
name: Browser E2E
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -27,44 +25,9 @@ jobs:
~/.cargo/registry
key: e2e-${{ runner.os }}-${{ hashFiles('Cargo.lock') }}
- name: Build
- name: Build ironclaw (libsql)
run: cargo build --no-default-features --features libsql
- name: Upload binary
uses: actions/upload-artifact@v4
with:
name: ironclaw-e2e-binary
path: target/debug/ironclaw
retention-days: 1
# ── Step 2: run test slices in parallel ───────────────────────────────────
test:
name: E2E (${{ matrix.group }})
needs: build
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- group: core
files: "tests/e2e/scenarios/test_connection.py tests/e2e/scenarios/test_chat.py tests/e2e/scenarios/test_sse_reconnect.py tests/e2e/scenarios/test_html_injection.py"
- group: features
files: "tests/e2e/scenarios/test_skills.py tests/e2e/scenarios/test_tool_approval.py"
- group: extensions
files: "tests/e2e/scenarios/test_extensions.py"
steps:
- uses: actions/checkout@v6
- name: Download binary
uses: actions/download-artifact@v4
with:
name: ironclaw-e2e-binary
path: target/debug/
- name: Make binary executable
run: chmod +x target/debug/ironclaw
- uses: actions/setup-python@v5
with:
python-version: "3.12"
@@ -75,26 +38,13 @@ jobs:
pip install -e .
playwright install --with-deps chromium
- name: Run E2E tests (${{ matrix.group }})
run: pytest ${{ matrix.files }} -v --timeout=120
- name: Run E2E tests
run: pytest tests/e2e/ -v -x --timeout=120
- name: Upload screenshots on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-screenshots-${{ matrix.group }}
name: e2e-screenshots
path: tests/e2e/screenshots/
if-no-files-found: ignore
# ── Roll-up for branch protection ────────────────────────────────────────
e2e:
name: E2E Tests
runs-on: ubuntu-latest
if: always()
needs: [test]
steps:
- run: |
if [[ "${{ needs.test.result }}" != "success" ]]; then
echo "One or more E2E jobs failed"
exit 1
fi
-107
View File
@@ -1,107 +0,0 @@
name: Regression Test Check
on:
pull_request:
jobs:
regression-test:
name: Regression test enforcement
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Check for regression tests
env:
PR_TITLE: ${{ github.event.pull_request.title }}
PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
run: |
set -euo pipefail
BASE_REF="origin/${{ github.event.pull_request.base.ref }}"
# --- 1. Is this a fix PR? Check title first, then commit messages ---
IS_FIX=false
if grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$PR_TITLE"; then
IS_FIX=true
fi
if [ "$IS_FIX" = false ]; then
COMMITS=$(git log --format='%s' "${BASE_REF}..HEAD")
if grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$COMMITS"; then
IS_FIX=true
fi
fi
if [ "$IS_FIX" = false ]; then
echo "Not a fix PR — skipping regression test check."
exit 0
fi
echo "Fix PR detected."
# --- 2. Skip label or commit message marker ---
if grep -qF ',skip-regression-check,' <<< ",$PR_LABELS,"; then
echo "skip-regression-check label present — skipping."
exit 0
fi
COMMIT_BODIES=$(git log --format='%B' "${BASE_REF}..HEAD")
if grep -qF '[skip-regression-check]' <<< "$COMMIT_BODIES"; then
echo "[skip-regression-check] found in commit message — skipping."
exit 0
fi
# --- 3. Exempt static-only / docs-only changes ---
CHANGED_FILES=$(git diff --name-only "${BASE_REF}...HEAD")
if [ -z "$CHANGED_FILES" ]; then
echo "No changed files — skipping."
exit 0
fi
ALL_EXEMPT=true
while IFS= read -r file; do
case "$file" in
src/channels/web/static/*) ;;
*.md) ;;
*) ALL_EXEMPT=false; break ;;
esac
done <<< "$CHANGED_FILES"
if [ "$ALL_EXEMPT" = true ]; then
echo "All changes are static assets or docs — skipping."
exit 0
fi
# --- 4. Look for test changes ---
# Fast path: new test attributes or test modules in added lines.
if git diff "${BASE_REF}...HEAD" -U0 -- '*.rs' | grep -qE '^\+.*(#\[test\]|#\[tokio::test\]|#\[cfg\(test\)\]|mod tests)'; then
echo "Test changes found in .rs files."
exit 0
fi
# Whole-function context: detect edits inside existing test functions.
if git diff "${BASE_REF}...HEAD" -W -- '*.rs' | awk '
/^@@/ { if (has_test && has_add) { found=1; exit } has_test=0; has_add=0 }
/^ .*#\[test\]/ || /^ .*#\[tokio::test\]/ || /^ .*#\[cfg\(test\)\]/ || /^ .*mod tests/ { has_test=1 }
/^\+.*#\[test\]/ || /^\+.*#\[tokio::test\]/ || /^\+.*#\[cfg\(test\)\]/ || /^\+.*mod tests/ { has_test=1 }
/^\+[^+]/ { has_add=1 }
END { if (has_test && has_add) found=1; exit !found }
'; then
echo "Test changes found in existing test functions."
exit 0
fi
if grep -qE '^tests/' <<< "$CHANGED_FILES"; then
echo "Test file changes found under tests/."
exit 0
fi
# --- 5. No tests found ---
echo "::warning::This PR looks like a bug fix but contains no test changes. Every fix should include a regression test. Add a #[test] or #[tokio::test], or apply the 'skip-regression-check' label if not feasible."
exit 1
+2 -12
View File
@@ -413,9 +413,6 @@ jobs:
- build-wasm-extensions
if: ${{ always() && needs.host.result == 'success' && needs.build-wasm-extensions.result == 'success' }}
runs-on: "ubuntu-22.04"
permissions:
contents: write
pull-requests: write
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
@@ -448,7 +445,7 @@ jobs:
fi
done
done < "$CHECKSUMS"
- name: Create PR with updated manifests
- name: Commit updated manifests
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
@@ -456,15 +453,8 @@ jobs:
if git diff --cached --quiet; then
echo "No manifest changes to commit"
else
BRANCH="chore/update-checksums-$(date +%s)"
git checkout -b "$BRANCH"
git commit -m "chore: update WASM artifact SHA256 checksums [skip ci]"
git push origin "$BRANCH"
gh pr create \
--title "chore: update WASM artifact SHA256 checksums" \
--body "Auto-generated by release CI. Updates SHA256 checksums in registry manifests to match the released WASM artifacts." \
--base main \
--head "$BRANCH"
git push
fi
announce:
-473
View File
@@ -1,473 +0,0 @@
name: Staging CI (Batched)
on:
schedule:
- cron: "0 * * * *" # Every 60 minutes
workflow_dispatch:
inputs:
force:
description: "Force run even if no new commits"
type: boolean
default: false
skip_claude_gate:
description: "Skip Claude review gate (bypass blocking findings)"
type: boolean
default: false
permissions:
contents: write
issues: write
pull-requests: write
checks: read
concurrency:
group: staging-ci
cancel-in-progress: false # Let running suites finish
jobs:
# ── Check for new commits ──────────────────────────────────────
check-changes:
name: Check for new commits
runs-on: ubuntu-latest
outputs:
has_changes: ${{ steps.check.outputs.has_changes }}
current_head: ${{ steps.check.outputs.current_head }}
diff_range: ${{ steps.check.outputs.diff_range }}
steps:
- uses: actions/checkout@v6
with:
ref: staging
fetch-depth: 0
fetch-tags: true
- name: Check for changes since last tested
id: check
env:
FORCE_RUN: ${{ inputs.force }}
run: |
CURRENT_HEAD=$(git rev-parse HEAD)
echo "current_head=${CURRENT_HEAD}" >> "$GITHUB_OUTPUT"
if git rev-parse staging-tested >/dev/null 2>&1; then
LAST_TESTED=$(git rev-parse staging-tested)
else
LAST_TESTED=""
fi
DIFF_RANGE=""
if [ -n "$LAST_TESTED" ] && [ "$LAST_TESTED" = "$CURRENT_HEAD" ]; then
echo "No new commits since last tested (${CURRENT_HEAD})"
HAS_CHANGES=false
else
HAS_CHANGES=true
if [ -n "$LAST_TESTED" ]; then
COMMIT_COUNT=$(git rev-list --count "${LAST_TESTED}..HEAD")
echo "Found ${COMMIT_COUNT} new commit(s) since last tested"
DIFF_RANGE="${LAST_TESTED}..${CURRENT_HEAD}"
else
git fetch origin main
MERGE_BASE=$(git merge-base origin/main HEAD)
echo "First run -- reviewing from merge-base ${MERGE_BASE}"
DIFF_RANGE="${MERGE_BASE}..${CURRENT_HEAD}"
fi
fi
# Force override from workflow_dispatch
if [ "$FORCE_RUN" = "true" ]; then
echo "Force run requested"
HAS_CHANGES=true
if [ -z "$DIFF_RANGE" ]; then
DIFF_RANGE="${CURRENT_HEAD}..${CURRENT_HEAD}"
fi
fi
echo "has_changes=${HAS_CHANGES}" >> "$GITHUB_OUTPUT"
echo "diff_range=${DIFF_RANGE}" >> "$GITHUB_OUTPUT"
# ── Run full test suite ──────────────────────────────────────────
tests:
name: Test Suite
needs: check-changes
if: needs.check-changes.outputs.has_changes == 'true'
uses: ./.github/workflows/test.yml
# ── Run E2E browser tests ────────────────────────────────────────
e2e:
name: E2E Browser Tests
needs: check-changes
if: needs.check-changes.outputs.has_changes == 'true'
uses: ./.github/workflows/e2e.yml
# ── Create promotion PR (triggers claude-review.yml on the PR) ──
create-promotion-pr:
name: Create Promotion PR
needs: check-changes
if: needs.check-changes.outputs.has_changes == 'true'
runs-on: ubuntu-latest
outputs:
pr_number: ${{ steps.create-pr.outputs.pr_number }}
promotion_branch: ${{ steps.branch.outputs.branch }}
steps:
- uses: actions/checkout@v6
with:
ref: staging
fetch-depth: 0
- name: Generate GitHub App token
id: app-token
continue-on-error: true
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }}
private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }}
- name: Set token
id: token
run: |
if [ -n "${{ steps.app-token.outputs.token }}" ]; then
echo "token=${{ steps.app-token.outputs.token }}" >> "$GITHUB_OUTPUT"
else
echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT"
fi
- name: Check if staging is ahead of main
id: ahead-check
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: |
git fetch origin main
AHEAD=$(git rev-list --count origin/main..origin/staging)
echo "commits_ahead=${AHEAD}" >> "$GITHUB_OUTPUT"
if [ "$AHEAD" -eq 0 ]; then
echo "Staging is not ahead of main. Nothing to promote."
else
echo "Staging is ${AHEAD} commits ahead of main."
fi
- name: Create promotion branch
id: branch
if: steps.ahead-check.outputs.commits_ahead != '0'
run: |
SHORT_SHA=$(echo "${{ needs.check-changes.outputs.current_head }}" | cut -c1-8)
BRANCH="staging-promote/${SHORT_SHA}-${{ github.run_id }}"
git checkout -b "$BRANCH"
git push origin "$BRANCH"
echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
echo "Created promotion branch: ${BRANCH}"
- name: Find base branch
id: find-base
if: steps.ahead-check.outputs.commits_ahead != '0'
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: |
# Find the newest open promotion PR with a staging-promote/* head branch
LATEST=$(gh pr list --label staging-promotion --state open \
--json headRefName,createdAt \
--jq '[.[] | select(.headRefName | startswith("staging-promote/"))] | sort_by(.createdAt) | last | .headRefName // empty')
if [ -n "$LATEST" ]; then
echo "base=${LATEST}" >> "$GITHUB_OUTPUT"
echo "Chaining onto existing promotion branch: ${LATEST}"
else
echo "base=main" >> "$GITHUB_OUTPUT"
echo "No existing promotion PR — targeting main"
fi
- name: Create promotion PR
id: create-pr
if: steps.ahead-check.outputs.commits_ahead != '0'
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
run: |
RANGE="${{ needs.check-changes.outputs.diff_range }}"
TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC")
BRANCH="${{ steps.branch.outputs.branch }}"
BASE="${{ steps.find-base.outputs.base }}"
PR_URL=$(gh pr create \
--base "$BASE" \
--head "$BRANCH" \
--title "chore: promote staging to main (${TIMESTAMP})" \
--body "## Auto-promotion from staging CI
**Batch range:** \`${RANGE}\`
**Promotion branch:** \`${BRANCH}\`
**Base:** \`${BASE}\`
**Triggered by:** Staging CI batch at ${TIMESTAMP}
Waiting for gates:
- Tests: pending
- E2E: pending
- Claude Code review: pending (will post comments on this PR)
---
*Auto-created by staging-ci workflow*" \
--label "staging-promotion")
PR_NUM=$(echo "$PR_URL" | grep -oE '[0-9]+$')
echo "pr_number=${PR_NUM}" >> "$GITHUB_OUTPUT"
echo "Created promotion PR #${PR_NUM}"
# ── Gate: wait for review, process findings, merge or block ─────
gate:
name: Staging Gate
needs: [check-changes, tests, e2e, create-promotion-pr]
if: >
always() &&
needs.check-changes.outputs.has_changes == 'true' &&
needs.tests.result == 'success' &&
needs.e2e.result == 'success' &&
needs.create-promotion-pr.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 25
outputs:
gate_passed: ${{ steps.evaluate.outputs.passed }}
steps:
- uses: actions/checkout@v6
with:
ref: staging
fetch-depth: 1
- name: Generate GitHub App token
id: app-token
continue-on-error: true
uses: actions/create-github-app-token@v2
with:
app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }}
private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }}
- name: Set token
id: token
run: |
if [ -n "${{ steps.app-token.outputs.token }}" ]; then
echo "token=${{ steps.app-token.outputs.token }}" >> "$GITHUB_OUTPUT"
else
echo "token=${{ github.token }}" >> "$GITHUB_OUTPUT"
fi
- name: Wait for Claude review job
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }}
REPO: ${{ github.repository }}
run: |
if [ -z "$PR_NUMBER" ]; then
echo "No PR number — skipping wait"
exit 0
fi
PR_SHA=$(gh pr view "$PR_NUMBER" --json headRefOid --jq '.headRefOid' || echo "")
if [ -z "$PR_SHA" ]; then
echo "::warning::Could not get PR head SHA"
exit 0
fi
echo "Polling for Claude Code Review job on PR #${PR_NUMBER} (SHA: ${PR_SHA})..."
TIMEOUT=1200 # 20 minutes
ELAPSED=0
INTERVAL=30
while [ "$ELAPSED" -lt "$TIMEOUT" ]; do
STATUS=$(gh api "repos/${REPO}/commits/${PR_SHA}/check-runs" \
--jq '[.check_runs[] | select(.name == "Claude Code Review") | .conclusion // .status] | first // "pending"' 2>/dev/null || echo "pending")
if [ "$STATUS" = "success" ] || [ "$STATUS" = "failure" ] || [ "$STATUS" = "cancelled" ]; then
echo "Claude review job completed with status: ${STATUS} (${ELAPSED}s)"
exit 0
fi
echo "Claude review status: ${STATUS} (${ELAPSED}s elapsed)"
sleep "$INTERVAL"
ELAPSED=$((ELAPSED + INTERVAL))
done
echo "::warning::Claude review job not completed after ${TIMEOUT}s"
- name: Process Claude review comments and create issues
id: process-findings
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }}
REPO: ${{ github.repository }}
run: |
HAS_BLOCKING=false
ISSUES_CREATED=0
if [ -z "$PR_NUMBER" ]; then
echo "No PR — skipping finding processing"
echo "has_blocking=false" >> "$GITHUB_OUTPUT"
exit 0
fi
# Check for "No issues found" first (clean pass)
NO_ISSUES=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
--jq '[.[] | select(.user.login == "claude[bot]") | select(.body | test("No issues found"))] | length' 2>/dev/null || echo "0")
if [ "$NO_ISSUES" -gt 0 ]; then
echo "Claude review found no issues — gate passes"
echo "has_blocking=false" >> "$GITHUB_OUTPUT"
exit 0
fi
# Get the last Claude comment that contains findings
JQ_FILTER='[.[] | select(.user.login == "claude[bot]") | select(.body | test("Found [0-9]+ issue"))] | last'
BODY=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
--jq "${JQ_FILTER} | .body // empty" 2>/dev/null || echo "")
COMMENT_URL=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
--jq "${JQ_FILTER} | .html_url // empty" 2>/dev/null || echo "")
if [ -z "$BODY" ]; then
echo "::warning::No Claude review comment found for PR #${PR_NUMBER} — treating as blocking"
echo "has_blocking=true" >> "$GITHUB_OUTPUT"
exit 0
fi
# Parse [SEVERITY:CONFIDENCE] tags from each numbered finding
# Matrix: CRITICAL always→issue, ≥80→block. HIGH ≥50→issue. MEDIUM ≥80→issue. LOW ≥80→issue.
# Use process substitution so variables propagate to parent shell
while read -r line; do
TAG=$(echo "$line" | grep -oE '^\[(CRITICAL|HIGH|MEDIUM|LOW):[0-9]+\]')
SEVERITY=$(echo "$TAG" | sed 's/\[\(.*\):\(.*\)\]/\1/')
CONFIDENCE=$(echo "$TAG" | sed 's/\[\(.*\):\(.*\)\]/\2/')
DESC=$(echo "$line" | sed "s/\[${SEVERITY}:${CONFIDENCE}\] *//" | head -1)
echo "Found: [${SEVERITY}:${CONFIDENCE}] ${DESC}"
# Check if blocking (CRITICAL ≥80)
if [ "$SEVERITY" = "CRITICAL" ] && [ "$CONFIDENCE" -ge 80 ]; then
HAS_BLOCKING=true
fi
# Determine if this should create an issue
CREATE_ISSUE=false
case "$SEVERITY" in
CRITICAL) CREATE_ISSUE=true ;;
HIGH) [ "$CONFIDENCE" -ge 50 ] && CREATE_ISSUE=true ;;
MEDIUM) [ "$CONFIDENCE" -ge 80 ] && CREATE_ISSUE=true ;;
LOW) [ "$CONFIDENCE" -ge 80 ] && CREATE_ISSUE=true ;;
esac
if [ "$CREATE_ISSUE" = "true" ]; then
case "$SEVERITY" in
CRITICAL) LABELS="bug,risk: high,staging-ci-review" ;;
HIGH) LABELS="bug,risk: medium,staging-ci-review" ;;
MEDIUM) LABELS="risk: medium,staging-ci-review" ;;
LOW) LABELS="risk: low,staging-ci-review" ;;
esac
TITLE=$(echo "$DESC" | cut -c1-80)
{
echo "## [${SEVERITY}:${CONFIDENCE}] Issue Found by Staging CI Review"
echo ""
echo "**Severity:** ${SEVERITY}"
echo "**Confidence:** ${CONFIDENCE}/100"
echo "**PR comment:** ${COMMENT_URL}"
echo ""
echo "### Description"
echo "$DESC"
echo ""
echo "---"
echo "*Auto-created by staging-ci Claude Code review*"
} > /tmp/issue-body.md
if gh issue create \
--title "[${SEVERITY}] ${TITLE}" \
--body-file /tmp/issue-body.md \
--label "${LABELS}"; then
ISSUES_CREATED=$((ISSUES_CREATED + 1))
else
echo "::warning::Failed to create issue for ${SEVERITY} finding"
fi
fi
done < <(echo "$BODY" | grep -oE '\[(CRITICAL|HIGH|MEDIUM|LOW):[0-9]+\].*')
echo "Created ${ISSUES_CREATED} issues"
echo "has_blocking=${HAS_BLOCKING}" >> "$GITHUB_OUTPUT"
- name: Evaluate gate
id: evaluate
env:
PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }}
SKIP_GATE: ${{ inputs.skip_claude_gate }}
HAS_BLOCKING: ${{ steps.process-findings.outputs.has_blocking }}
run: |
SKIP_INPUT="$SKIP_GATE"
if [ "$HAS_BLOCKING" = "true" ]; then
echo "::warning::Claude review found blocking issues (CRITICAL ≥80 confidence)"
if [ "$SKIP_INPUT" = "true" ]; then
echo "::warning::Gate overridden by skip_claude_gate workflow input"
echo "passed=true" >> "$GITHUB_OUTPUT"
else
echo "::error::Blocking promotion due to CRITICAL findings (≥80 confidence)"
echo "::error::PR #${PR_NUMBER} left open with review comments"
echo "passed=false" >> "$GITHUB_OUTPUT"
exit 1
fi
else
echo "No blocking findings. Gate passed."
echo "passed=true" >> "$GITHUB_OUTPUT"
fi
- name: Merge promotion PR
id: merge
if: steps.evaluate.outputs.passed == 'true'
env:
GH_TOKEN: ${{ steps.token.outputs.token }}
PR_NUMBER: ${{ needs.create-promotion-pr.outputs.pr_number }}
run: |
if [ -n "$PR_NUMBER" ]; then
echo "Merging promotion PR #${PR_NUMBER}"
# Do NOT use --delete-branch: deleting a promotion branch closes
# any chained PRs that use it as their base (verified in ironclaw-ci-test).
# Stale promotion branches are cleaned up separately.
gh pr merge "$PR_NUMBER" --merge
echo "merged=true" >> "$GITHUB_OUTPUT"
fi
# ── Update tested tag (always, so next batch covers only new commits) ──
update-tag:
name: Update staging-tested tag
needs: [check-changes, tests, e2e, create-promotion-pr, gate]
if: >
always() &&
needs.check-changes.outputs.has_changes == 'true' &&
needs.tests.result == 'success' &&
needs.e2e.result == 'success' &&
needs.create-promotion-pr.result == 'success'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
ref: staging
fetch-depth: 1
- name: Update staging-tested tag
run: |
git tag -f staging-tested "${{ needs.check-changes.outputs.current_head }}"
git push origin staging-tested --force
echo "Updated staging-tested tag to ${{ needs.check-changes.outputs.current_head }}"
# ── Report ───────────────────────────────────────────────────────
report:
name: Staging CI Summary
needs: [check-changes, tests, e2e, create-promotion-pr, gate, update-tag]
if: always() && needs.check-changes.outputs.has_changes == 'true'
runs-on: ubuntu-latest
steps:
- name: Summary
run: |
echo "## Staging CI Batch Results" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "| Check | Result |" >> "$GITHUB_STEP_SUMMARY"
echo "|-------|--------|" >> "$GITHUB_STEP_SUMMARY"
echo "| Tests | ${{ needs.tests.result }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| E2E | ${{ needs.e2e.result }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| Promotion PR | ${{ needs.create-promotion-pr.result }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| Gate | ${{ needs.gate.result }} |" >> "$GITHUB_STEP_SUMMARY"
echo "| Tag Updated | ${{ needs.update-tag.result }} |" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "Range: ${{ needs.check-changes.outputs.diff_range }}" >> "$GITHUB_STEP_SUMMARY"
PR_NUM="${{ needs.create-promotion-pr.outputs.pr_number }}"
if [ -n "$PR_NUM" ]; then
echo "Promotion PR: #${PR_NUM}" >> "$GITHUB_STEP_SUMMARY"
fi
+13 -101
View File
@@ -1,6 +1,5 @@
name: Run Tests
on:
workflow_call:
pull_request:
push:
branches:
@@ -10,54 +9,6 @@ jobs:
tests:
name: Tests (${{ matrix.name }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- name: all-features
flags: "--features postgres,libsql,html-to-markdown"
- name: default
flags: ""
- name: libsql-only
flags: "--no-default-features --features libsql"
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-wasip2
- uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.name }}
- name: Install cargo-component
run: cargo install cargo-component --locked || true
- name: Build WASM channels (for integration tests)
run: ./scripts/build-wasm-extensions.sh --channels
- name: Run Tests
run: cargo test ${{ matrix.flags }} -- --nocapture
telegram-tests:
name: Telegram Channel Tests
if: >
github.event_name == 'push' ||
(github.event_name == 'pull_request' && github.base_ref != 'staging')
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Run Telegram Channel Tests
run: cargo test --manifest-path channels-src/telegram/Cargo.toml -- --nocapture
windows-build:
name: Windows Build (${{ matrix.name }})
if: >
github.event_name == 'push' ||
(github.event_name == 'pull_request' && github.base_ref != 'staging')
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
@@ -73,17 +24,16 @@ jobs:
uses: actions/checkout@v6
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
profile: minimal
- uses: Swatinem/rust-cache@v2
with:
key: windows-${{ matrix.name }}
- name: Check compilation
run: cargo check --all --benches --tests --examples ${{ matrix.flags }}
key: ${{ matrix.name }}
- name: Run Tests
run: cargo test ${{ matrix.flags }} -- --nocapture
wasm-wit-compat:
name: WASM WIT Compatibility
if: >
github.event_name == 'push' ||
(github.event_name == 'pull_request' && github.base_ref != 'staging')
telegram-tests:
name: Telegram Channel Tests
runs-on: ubuntu-latest
steps:
- name: Checkout repository
@@ -91,22 +41,13 @@ jobs:
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: wasm32-wasip2
profile: minimal
- uses: Swatinem/rust-cache@v2
with:
key: wasm-extensions
- name: Install cargo-component
run: cargo install cargo-component --locked || true
- name: Build all WASM extensions against current WIT
run: ./scripts/build-wasm-extensions.sh
- name: Instantiation test (host linker compatibility)
run: cargo test --all-features wit_compat -- --nocapture
- name: Run Telegram Channel Tests
run: cargo test --manifest-path channels-src/telegram/Cargo.toml -- --nocapture
docker-build:
name: Docker Build
if: >
github.event_name == 'push' ||
(github.event_name == 'pull_request' && github.base_ref != 'staging')
runs-on: ubuntu-latest
steps:
- name: Checkout repository
@@ -114,44 +55,15 @@ jobs:
- name: Build Docker image
run: docker build -t ironclaw-test:ci .
version-check:
name: Version Bump Check
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Check version bumps for changed extensions
env:
PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }}
run: ./scripts/check-version-bumps.sh
# Roll-up job for branch protection
run-tests:
name: Run Tests
runs-on: ubuntu-latest
if: always()
needs: [tests, telegram-tests, wasm-wit-compat, docker-build, windows-build, version-check]
needs: [tests, telegram-tests, docker-build]
steps:
- run: |
# Unit tests must always pass
if [[ "${{ needs.tests.result }}" != "success" ]]; then
echo "Unit tests failed"
if [[ "${{ needs.tests.result }}" != "success" || "${{ needs.telegram-tests.result }}" != "success" || "${{ needs.docker-build.result }}" != "success" ]]; then
echo "One or more jobs failed"
exit 1
fi
# Gated jobs: must pass on promotion PRs / push, skipped on developer PRs
for job in telegram-tests wasm-wit-compat docker-build windows-build version-check; do
case "$job" in
telegram-tests) result="${{ needs.telegram-tests.result }}" ;;
wasm-wit-compat) result="${{ needs.wasm-wit-compat.result }}" ;;
docker-build) result="${{ needs.docker-build.result }}" ;;
windows-build) result="${{ needs.windows-build.result }}" ;;
version-check) result="${{ needs.version-check.result }}" ;;
esac
if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
echo "$job failed"
exit 1
fi
done
+1 -7
View File
@@ -4,9 +4,8 @@
.env.*
!.env.example
# Claude Code worktrees and lock files
# Claude Code worktrees
.claude/worktrees/
.claude/scheduled_tasks.lock
# Sidecar tool data
.sidecar/
@@ -17,11 +16,6 @@ target/
# Benchmark results (local runs, not committed)
bench-results/
# Coverage reports (local runs, not committed)
/coverage/
# WASM build artifacts (loaded from disk, not bundled)
*.wasm
# Traces
trace_*.json
-101
View File
@@ -7,107 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- AWS Bedrock LLM provider via native Converse API with IAM and SSO auth support (feature-gated: `--features bedrock`)
## [0.16.1](https://github.com/nearai/ironclaw/compare/v0.16.0...v0.16.1) - 2026-03-06
### Fixed
- revert WASM artifact SHA256 checksums to null ([#627](https://github.com/nearai/ironclaw/pull/627))
## [0.16.0](https://github.com/nearai/ironclaw/compare/v0.15.0...v0.16.0) - 2026-03-06
### Added
- *(e2e)* extensions tab tests, CI parallelization, and 3 production bug fixes ([#584](https://github.com/nearai/ironclaw/pull/584))
- WASM extension versioning with WIT compat checks ([#592](https://github.com/nearai/ironclaw/pull/592))
- Add HMAC-SHA256 webhook signature validation for Slack ([#588](https://github.com/nearai/ironclaw/pull/588))
- restart ([#531](https://github.com/nearai/ironclaw/pull/531))
- merge http/web_fetch tools, add tool output stash for large responses ([#578](https://github.com/nearai/ironclaw/pull/578))
- integrate 13-dimension complexity scorer into smart routing ([#529](https://github.com/nearai/ironclaw/pull/529))
### Fixed
- *(llm)* fix reasoning model response parsing bugs ([#564](https://github.com/nearai/ironclaw/pull/564)) ([#580](https://github.com/nearai/ironclaw/pull/580))
- *(ci)* fix three coverage workflow failures ([#597](https://github.com/nearai/ironclaw/pull/597))
- Telegram channel accepts group messages from all users if owner_… ([#590](https://github.com/nearai/ironclaw/pull/590))
- *(ci)* anchor coverage/ gitignore rule to repo root ([#591](https://github.com/nearai/ironclaw/pull/591))
- *(security)* use OsRng for all security-critical key and token generation ([#519](https://github.com/nearai/ironclaw/pull/519))
- prevent concurrent memory hygiene passes and Windows file lock errors ([#535](https://github.com/nearai/ironclaw/pull/535))
- sort tool_definitions() for deterministic LLM tool ordering ([#582](https://github.com/nearai/ironclaw/pull/582))
- *(ci)* persist all cargo-llvm-cov env vars for E2E coverage ([#559](https://github.com/nearai/ironclaw/pull/559))
### Other
- *(llm)* complete response cache — set_model invalidation, stats logging, sync mutex ([#290](https://github.com/nearai/ironclaw/pull/290))
- add 29 E2E trace tests for issues #571-575 ([#593](https://github.com/nearai/ironclaw/pull/593))
- add 26 tests for multi-thread safety, db CRUD, concurrency, errors ([#442](https://github.com/nearai/ironclaw/pull/442))
- update WASM artifact SHA256 checksums [skip ci] ([#560](https://github.com/nearai/ironclaw/pull/560))
- add WIT compatibility tests for WASM extensions ([#586](https://github.com/nearai/ironclaw/pull/586))
- Trajectory benchmarks and e2e trace test rig ([#553](https://github.com/nearai/ironclaw/pull/553))
## [0.15.0](https://github.com/nearai/ironclaw/compare/v0.14.0...v0.15.0) - 2026-03-04
### Added
- *(oauth)* route callbacks through web gateway for hosted instances ([#555](https://github.com/nearai/ironclaw/pull/555))
- *(web)* show error details for failed tool calls ([#490](https://github.com/nearai/ironclaw/pull/490))
- *(extensions)* improve auth UX and add load-time validation ([#536](https://github.com/nearai/ironclaw/pull/536))
- add local-test skill and Dockerfile.test for web gateway testing ([#524](https://github.com/nearai/ironclaw/pull/524))
### Fixed
- *(security)* restrict query-token auth to SSE endpoints only ([#528](https://github.com/nearai/ironclaw/pull/528))
- *(ci)* flush profraw coverage data in E2E teardown ([#550](https://github.com/nearai/ironclaw/pull/550))
- *(wasm)* coerce string parameters to schema-declared types ([#498](https://github.com/nearai/ironclaw/pull/498))
- *(agent)* strip leaked [Called tool ...] text from responses ([#497](https://github.com/nearai/ironclaw/pull/497))
- *(web)* reset job list UI on restart failure ([#499](https://github.com/nearai/ironclaw/pull/499))
- *(security)* replace .unwrap() panics in pairing store with proper error handling ([#515](https://github.com/nearai/ironclaw/pull/515))
### Other
- Fix UTF-8 unsafe truncation in sandbox log capture ([#359](https://github.com/nearai/ironclaw/pull/359))
- enhance coverage with feature matrix, postgres, and E2E ([#523](https://github.com/nearai/ironclaw/pull/523))
## [0.14.0](https://github.com/nearai/ironclaw/compare/v0.13.1...v0.14.0) - 2026-03-04
### Added
- remove the okta tool ([#506](https://github.com/nearai/ironclaw/pull/506))
- add OAuth support for WASM tools in web gateway ([#489](https://github.com/nearai/ironclaw/pull/489))
- *(web)* fix jobs UI parity for non-sandbox mode ([#491](https://github.com/nearai/ironclaw/pull/491))
- *(workspace)* add TOOLS.md, BOOTSTRAP.md, and disk-to-DB import ([#477](https://github.com/nearai/ironclaw/pull/477))
### Fixed
- *(web)* mobile browser bar obscures chat input ([#508](https://github.com/nearai/ironclaw/pull/508))
- *(web)* assign unique thread_id to manual routine triggers ([#500](https://github.com/nearai/ironclaw/pull/500))
- *(web)* refresh routine UI after Run Now trigger ([#501](https://github.com/nearai/ironclaw/pull/501))
- *(skills)* use slug for skill download URL from ClawHub ([#502](https://github.com/nearai/ironclaw/pull/502))
- *(workspace)* thread document path through search results ([#503](https://github.com/nearai/ironclaw/pull/503))
- *(workspace)* import custom templates before seeding defaults ([#505](https://github.com/nearai/ironclaw/pull/505))
- use std::sync::RwLock in MessageTool to avoid runtime panic ([#411](https://github.com/nearai/ironclaw/pull/411))
- wire secrets store into all WASM runtime activation paths ([#479](https://github.com/nearai/ironclaw/pull/479))
### Other
- enforce regression tests for fix commits ([#517](https://github.com/nearai/ironclaw/pull/517))
- add code coverage with cargo-llvm-cov and Codecov ([#511](https://github.com/nearai/ironclaw/pull/511))
- Remove restart infrastructure, generalize WASM channel setup ([#493](https://github.com/nearai/ironclaw/pull/493))
## [0.13.1](https://github.com/nearai/ironclaw/compare/v0.13.0...v0.13.1) - 2026-03-02
### Added
- add Brave Web Search WASM tool ([#474](https://github.com/nearai/ironclaw/pull/474))
### Fixed
- *(web)* auto-scroll and Enter key completion for slash command autocomplete ([#475](https://github.com/nearai/ironclaw/pull/475))
- correct download URLs for telegram-mtproto and slack-tool extensions ([#470](https://github.com/nearai/ironclaw/pull/470))
## [0.13.0](https://github.com/nearai/ironclaw/compare/v0.12.0...v0.13.0) - 2026-03-02
### Added
+132 -165
View File
@@ -43,53 +43,34 @@ cargo test test_name
# Run with logging
RUST_LOG=ironclaw=debug cargo run
# Run integration tests (may require running services/DB)
cargo test --test workspace_integration
cargo test --test ws_gateway_integration
cargo test --test heartbeat_integration
# Run E2E tests (Python/Playwright — requires a running ironclaw instance)
# See tests/e2e/CLAUDE.md for full setup instructions
cd tests/e2e
python -m venv .venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate
pip install -e .
playwright install chromium
pytest scenarios/ # all scenarios
pytest scenarios/test_chat.py # specific scenario
```
### Test Tiers
| Tier | Command | What runs | External deps |
|------|---------|-----------|---------------|
| Unit | `cargo test` | All `mod tests` + self-contained integration tests | None |
| Integration | `cargo test --features integration` | + PostgreSQL-dependent tests | Running PostgreSQL |
| Live | `cargo test --features integration -- --ignored` | + LLM-dependent tests | PostgreSQL + LLM API keys |
Run `bash scripts/check-boundaries.sh` to verify test tier gating and other architecture rules.
## Project Structure
```
src/
├── lib.rs # Library root, module declarations
├── main.rs # Entry point, CLI args, startup
├── app.rs # App startup orchestration (channel wiring, DB init)
├── bootstrap.rs # Base directory resolution (~/.ironclaw), early .env loading
├── settings.rs # User settings persistence (~/.ironclaw/settings.json)
├── service.rs # OS service management (launchd/systemd daemon install)
├── tracing_fmt.rs # Custom tracing formatter
├── util.rs # Shared utilities
├── config/ # Configuration from env vars (split by subsystem)
│ ├── mod.rs # Re-exports all config types; top-level Config struct
│ ├── agent.rs, llm.rs, channels.rs, database.rs, sandbox.rs, skills.rs
│ ├── heartbeat.rs, routines.rs, safety.rs, embeddings.rs, wasm.rs
│ ├── tunnel.rs # Tunnel provider config (TUNNEL_PROVIDER, TUNNEL_URL, etc.)
│ └── secrets.rs, hygiene.rs, builder.rs, helpers.rs
├── config.rs # Configuration from env vars
├── error.rs # Error types (thiserror)
├── agent/ # Core agent loop, dispatcher, scheduler, sessions — see src/agent/CLAUDE.md
├── agent/ # Core agent logic
│ ├── agent_loop.rs # Main Agent struct, message handling loop
│ ├── router.rs # MessageIntent classification
│ ├── scheduler.rs # Parallel job scheduling
│ ├── worker.rs # Per-job execution with LLM reasoning
│ ├── self_repair.rs # Stuck job detection and recovery
│ ├── heartbeat.rs # Proactive periodic execution
│ ├── session.rs # Session/thread/turn model with state machine
│ ├── session_manager.rs # Thread/session lifecycle management
│ ├── compaction.rs # Context window management with turn summarization
│ ├── context_monitor.rs # Memory pressure detection
│ ├── undo.rs # Turn-based undo/redo with checkpoints
│ ├── submission.rs # Submission parsing (undo, redo, compact, clear, etc.)
│ ├── dispatcher.rs # Skill-aware job dispatching
│ ├── task.rs # Sub-task execution framework
│ ├── routine.rs # Routine types (Trigger, Action, Guardrails)
│ └── routine_engine.rs # Routine execution (cron ticker, event matcher)
├── channels/ # Multi-channel input
│ ├── channel.rs # Channel trait, IncomingMessage, OutgoingResponse
@@ -102,60 +83,21 @@ src/
│ │ ├── overlay.rs # Approval overlays
│ │ └── composer.rs # Message composition
│ ├── http.rs # HTTP webhook (axum) with secret validation
│ ├── webhook_server.rs # Unified HTTP server composing all webhook routes
│ ├── repl.rs # Simple REPL (for testing)
│ ├── web/ # Web gateway (browser UI) — see src/channels/web/CLAUDE.md
│ ├── web/ # Web gateway (browser UI)
│ │ ├── mod.rs # Gateway builder, startup
│ │ ├── server.rs # Axum router, 40+ API endpoints
│ │ ├── sse.rs # SSE broadcast manager
│ │ ├── ws.rs # WebSocket gateway + connection tracking
│ │ ├── types.rs # Request/response types, SseEvent enum
│ │ ├── auth.rs # Bearer token auth middleware
│ │ ├── log_layer.rs # Tracing layer for log streaming
│ │ └── static/ # HTML, CSS, JS (single-page app)
│ └── wasm/ # WASM channel runtime
│ ├── mod.rs
│ ├── bundled.rs # Bundled channel discovery
│ ├── capabilities.rs # Channel-specific capabilities (HTTP endpoint, emit rate)
│ ├── error.rs # WASM channel error types
│ ├── runtime.rs # WASM channel execution runtime
│ └── wrapper.rs # Channel trait wrapper for WASM modules
├── cli/ # CLI subcommands (clap)
│ ├── mod.rs # Cli struct, Command enum (run/onboard/config/tool/registry/mcp/memory/pairing/service/doctor/status/completion)
│ ├── config.rs # config list/get/set subcommands
│ ├── tool.rs # tool install/list/remove subcommands
│ ├── registry.rs # registry list/install subcommands
│ ├── mcp.rs # mcp add/auth/list/test subcommands
│ ├── memory.rs # memory search/read/write subcommands
│ ├── pairing.rs # pairing list/approve subcommands
│ ├── service.rs # service install/start/stop subcommands
│ ├── doctor.rs # Active health diagnostics
│ ├── status.rs # System health/status display
│ ├── completion.rs # Shell completion script generation
│ └── oauth_defaults.rs # Default OAuth redirect URIs
├── registry/ # Extension registry catalog
│ ├── mod.rs # Public API; re-exports RegistryCatalog, RegistryInstaller, manifest types
│ ├── manifest.rs # ExtensionManifest, ArtifactSpec, BundleDefinition types
│ ├── catalog.rs # RegistryCatalog: load from filesystem and embedded JSON
│ ├── installer.rs # RegistryInstaller: download, verify, install WASM artifacts
│ ├── artifacts.rs # Artifact download and caching
│ └── embedded.rs # Catalog compiled into binary at build time (via build.rs)
├── hooks/ # Lifecycle hooks for intercepting agent operations
│ ├── mod.rs # 6 HookPoints: BeforeInbound, BeforeToolCall, BeforeOutbound, OnSessionStart, OnSessionEnd, TransformResponse
│ ├── hook.rs # Hook trait, HookContext, HookEvent, HookOutcome, HookFailureMode
│ ├── registry.rs # HookRegistry: register, prioritize, execute hooks
│ └── bundled.rs # Built-in hooks: rule-based filters, webhook forwarders, HookBundleConfig
├── tunnel/ # Tunnel abstraction for public internet exposure
│ ├── mod.rs # Tunnel trait, TunnelProviderConfig, create_tunnel() factory
│ ├── cloudflare.rs # CloudflareTunnel (cloudflared binary)
│ ├── ngrok.rs # NgrokTunnel
│ ├── tailscale.rs # TailscaleTunnel (serve/funnel modes)
│ ├── custom.rs # CustomTunnel (arbitrary command with {host}/{port})
│ └── none.rs # NoneTunnel (local-only, no exposure)
├── observability/ # Pluggable event/metric recording
│ ├── mod.rs # create_observer() factory, ObservabilityConfig
│ ├── traits.rs # Observer trait, ObserverEvent, ObserverMetric
│ ├── noop.rs # NoopObserver (zero overhead, default)
│ ├── log.rs # LogObserver (tracing-based)
│ └── multi.rs # MultiObserver (fan-out to multiple backends)
├── orchestrator/ # Internal HTTP API for sandbox containers
│ ├── mod.rs
│ ├── api.rs # Axum endpoints (LLM proxy, events, prompts)
@@ -173,30 +115,34 @@ src/
│ ├── sanitizer.rs # Pattern detection, content escaping
│ ├── validator.rs # Input validation (length, encoding, patterns)
│ ├── policy.rs # PolicyRule system with severity/actions
── leak_detector.rs # Secret detection (API keys, tokens, etc.)
│ └── credential_detect.rs # HTTP request credential detection (headers, URL params)
── leak_detector.rs # Secret detection (API keys, tokens, etc.)
├── llm/ # Multi-provider LLM integration — see src/llm/CLAUDE.md
├── llm/ # LLM integration (multi-provider)
│ ├── mod.rs # Provider factory, LlmBackend enum
│ ├── provider.rs # LlmProvider trait, message types
│ ├── nearai_chat.rs # NEAR AI Chat Completions provider (session token + API key auth)
│ ├── reasoning.rs # Planning, tool selection, evaluation
│ ├── session.rs # Session token management with auto-renewal
│ ├── circuit_breaker.rs # Circuit breaker for provider failures
│ ├── retry.rs # Retry with exponential backoff
│ ├── failover.rs # Multi-provider failover chain
│ ├── response_cache.rs # LLM response caching
│ ├── costs.rs # Token cost tracking
│ └── rig_adapter.rs # Rig framework adapter
├── tools/ # Extensible tool system
│ ├── tool.rs # Tool trait, ToolOutput, ToolError
│ ├── registry.rs # ToolRegistry for discovery
│ ├── sandbox.rs # Process-based sandbox (stub, superseded by wasm/)
│ ├── rate_limiter.rs # Shared sliding-window rate limiter for built-in and WASM tools
│ ├── builtin/ # Built-in tools
│ │ ├── echo.rs, time.rs, json.rs, http.rs
│ │ ├── web_fetch.rs # GET URL → clean Markdown (readability + html-to-md conversion)
│ │ ├── file.rs # ReadFile, WriteFile, ListDir, ApplyPatch
│ │ ├── shell.rs # Shell command execution
│ │ ├── memory.rs # Memory tools (search, write, read, tree)
│ │ ├── message.rs # MessageTool: agent proactively messages users on any channel
│ │ ├── job.rs # CreateJob, ListJobs, JobStatus, CancelJob
│ │ ├── routine.rs # routine_create/list/update/delete/history
│ │ ├── extension_tools.rs # Extension install/auth/activate/remove
│ │ ├── skill_tools.rs # skill_list/search/install/remove tools
│ │ ├── secrets_tools.rs # secret_list/secret_delete (zero-exposure: no values exposed)
│ │ ├── html_converter.rs # HTML→Markdown via readability + html-to-markdown-rs
│ │ ├── path_utils.rs # Shared path validation/canonicalization helpers
│ │ └── marketplace.rs, ecommerce.rs, taskrabbit.rs, restaurant.rs (stubs)
│ ├── builder/ # Dynamic tool building
│ │ ├── core.rs # BuildRequirement, SoftwareType, Language
@@ -205,8 +151,7 @@ src/
│ │ └── validation.rs # WASM validation
│ ├── mcp/ # Model Context Protocol
│ │ ├── client.rs # MCP client over HTTP
│ │ ── protocol.rs # JSON-RPC types
│ │ └── session.rs # MCP session management (Mcp-Session-Id header, per-server state)
│ │ ── protocol.rs # JSON-RPC types
│ └── wasm/ # Full WASM sandbox (wasmtime)
│ ├── runtime.rs # Module compilation and caching
│ ├── wrapper.rs # Tool trait wrapper for WASM modules
@@ -216,10 +161,13 @@ src/
│ ├── credential_injector.rs # Safe credential injection
│ ├── loader.rs # WASM tool discovery from filesystem
│ ├── rate_limiter.rs # Per-tool rate limiting
│ ├── error.rs # WASM-specific error types
│ └── storage.rs # Linear memory persistence
├── db/ # Dual-backend persistence (PostgreSQL + libSQL) — see src/db/CLAUDE.md
├── db/ # Database abstraction layer
│ ├── mod.rs # Database trait (~60 async methods)
│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository)
│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite)
│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent)
├── workspace/ # Persistent memory system (OpenClaw-inspired)
│ ├── mod.rs # Workspace struct, memory operations
@@ -257,11 +205,9 @@ src/
│ └── allowlist.rs # DomainAllowlist validation
├── secrets/ # Secrets management
│ ├── mod.rs # SecretsStore trait, public API
│ ├── types.rs # Core types (Secret, SecretRef, SecretMetadata)
│ ├── crypto.rs # AES-256-GCM encryption
│ ├── keychain.rs # OS keychain integration (macOS Keychain, GNOME Keyring) for master key
│ └── store.rs # Encrypted secret storage
│ ├── store.rs # Secret storage
│ └── types.rs # Credential types
├── setup/ # Onboarding wizard (spec: src/setup/README.md)
│ ├── mod.rs # Entry point, check_onboard_needed()
@@ -281,11 +227,6 @@ src/
└── history/ # Persistence
├── store.rs # PostgreSQL repositories
└── analytics.rs # Aggregation queries (JobStats, ToolStats)
tests/
├── *.rs # Integration tests (workspace, heartbeat, WS gateway, pairing, etc.)
├── test-pages/ # HTML→Markdown conversion fixtures (CNN, Medium, Yahoo)
└── e2e/ # Python/Playwright E2E scenarios (see tests/e2e/CLAUDE.md)
```
## Key Patterns
@@ -306,16 +247,13 @@ When designing new features or systems, always prefer generic/extensible archite
- Use `RwLock` for concurrent read/write access
### Traits for Extensibility
- `Database` - Add new database backends (must implement all ~78 methods)
- `Database` - Add new database backends (must implement all ~60 methods)
- `Channel` - Add new input sources
- `Tool` - Add new capabilities
- `LlmProvider` - Add new LLM backends
- `SuccessEvaluator` - Custom evaluation logic
- `EmbeddingProvider` - Add embedding backends (workspace search)
- `NetworkPolicyDecider` - Custom network access policies for sandbox containers
- `Hook` - Lifecycle hook at 6 interception points (BeforeInbound, BeforeToolCall, BeforeOutbound, OnSessionStart, OnSessionEnd, TransformResponse)
- `Observer` - Observability backend (noop/log/multi; future: OpenTelemetry, Prometheus)
- `Tunnel` - Tunnel provider for public internet exposure
### Tool Implementation
```rust
@@ -383,31 +321,13 @@ cargo check --all-features # all features
```
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
**Regression test with every fix:** Every bug fix must include a test that would have caught the bug. Add a `#[test]` or `#[tokio::test]` that reproduces the original failure. Exempt: changes limited to `src/channels/web/static/` or `.md` files. Use `[skip-regression-check]` in commit message or PR label if genuinely not feasible. The `commit-msg` hook and CI workflow enforce this automatically.
**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate.
**Transaction safety:** Multi-step database operations (INSERT+INSERT, UPDATE+DELETE, read-then-write) MUST be wrapped in a transaction. Never assume sequential calls are atomic. Before committing DB code, ask: "If this crashes between step N and N+1, is the database consistent?" If not, wrap in a transaction. This applies to both postgres and libsql backends.
**UTF-8 string safety:** Never use byte-index slicing (`&s[..n]`) on user-supplied or external strings — it panics on multi-byte characters. Use `is_char_boundary()` to walk backwards from the desired length, or iterate with `char_indices()`. Grep for `[..` in changed files to catch violations.
**Case-insensitive comparisons:** When comparing user-supplied strings (file paths, media types, extension names), always normalize to lowercase first with `.to_ascii_lowercase()`. On case-insensitive filesystems (macOS, Windows), path comparisons must be case-insensitive. File extension checks (`.png`, `.jpg`) and media type checks (`image/jpeg`) are common offenders.
**Decorator/wrapper trait delegation:** When adding a new method to `LlmProvider` (or any trait with decorator wrappers), you MUST update ALL wrapper types to delegate to their inner provider. Grep for `impl LlmProvider for` to find all implementations. Add a test that exercises the method through the full provider chain (`build_provider_chain()`), not just the base impl.
**Sensitive data in logs & events:** Tool parameters and outputs MUST be redacted before logging or broadcasting via SSE/WebSocket. Use `redact_params()` before any `tracing::info!`, `JobEvent`, or SSE emission that includes tool call data. Never log raw parameters from tool calls.
**Test temporary files:** Use the `tempfile` crate for test directories/files. Never hardcode `/tmp/...` paths — they collide in parallel test runs and break on non-Unix platforms.
**Trust boundaries in multi-process architecture:** Data from worker containers is untrusted. The orchestrator MUST validate: tool domain (never execute `Container`-domain tools on the host), nesting depth (server-side tracking, not client-supplied), and parameter sensitivity (redact before logging/broadcasting).
**Mechanical verification before committing:** Run these checks on changed files before committing:
- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
- `grep -rn 'super::' <files>` -- use `crate::` imports
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
- Fix commits must include regression tests (enforced by `commit-msg` hook; bypass with `[skip-regression-check]`)
- Run `scripts/pre-commit-safety.sh` to catch UTF-8, case-sensitivity, hardcoded /tmp, and logging issues
## Configuration
@@ -483,47 +403,99 @@ SKILLS_AUTO_DISCOVER=true # Scan skill directories on startup
# Tinfoil private inference
TINFOIL_API_KEY=... # Required when LLM_BACKEND=tinfoil
TINFOIL_MODEL=kimi-k2-5 # Default model
# AWS Bedrock (native Converse API, requires --features bedrock)
# LLM_BACKEND=bedrock
# BEDROCK_REGION=us-east-1 # AWS region
# BEDROCK_MODEL=anthropic.claude-opus-4-6-v1 # Required model ID
# BEDROCK_CROSS_REGION=us # Cross-region prefix (us/eu/apac/global)
# AWS_PROFILE=my-profile # Named profile (SSO/assume-role)
# Tunnel (public internet exposure for webhooks)
TUNNEL_URL=https://abc123.ngrok.io # Static public URL (manual tunnel)
# Or use a managed tunnel provider:
TUNNEL_PROVIDER=none # none (default), cloudflare, tailscale, ngrok, custom
TUNNEL_CF_TOKEN=... # Required for TUNNEL_PROVIDER=cloudflare
TUNNEL_NGROK_TOKEN=... # Required for TUNNEL_PROVIDER=ngrok
# TUNNEL_NGROK_DOMAIN=... # Custom domain (paid ngrok plan)
# TUNNEL_TS_FUNNEL=true # Use tailscale funnel (public) vs serve (tailnet)
TUNNEL_CUSTOM_COMMAND=... # Command with {host}/{port} for custom providers
# Observability backend
OBSERVABILITY_BACKEND=none # none/noop (default) or log
```
### LLM Providers
Backends: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, `tinfoil`, `bedrock` (requires `--features bedrock`) — set via `LLM_BACKEND`. See [src/llm/CLAUDE.md](src/llm/CLAUDE.md) for per-provider auth and configuration details.
IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, and `tinfoil`.
**AWS Bedrock** -- Uses the native Converse API via `aws-sdk-bedrockruntime`. Requires `--features bedrock` at build time (not included in default features due to heavy AWS SDK dependencies). Supports standard AWS auth methods: IAM credentials (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`), SSO profiles (`AWS_PROFILE`), and instance roles. Configure with `BEDROCK_REGION` (default: `us-east-1`), `BEDROCK_MODEL` (required, e.g., `anthropic.claude-opus-4-6-v1`), and `BEDROCK_CROSS_REGION` (optional: `us`, `eu`, `apac`, `global` for cross-region inference profiles). The SDK credential chain resolves auth automatically from the environment.
**NEAR AI** -- Uses the Chat Completions API with dual auth support. Session token auth (default): authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google), base URL defaults to `https://private.near.ai`. API key auth: set `NEARAI_API_KEY` (from `cloud.near.ai`), base URL defaults to `https://cloud-api.near.ai`. Both modes use the same Chat Completions endpoint. Tool messages are flattened to plain text for compatibility. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment.
**NEAR AI Cloud** -- Uses the OpenAI-compatible Chat Completions API (`https://cloud-api.near.ai/v1/chat/completions`). Authenticates with API keys from `cloud.near.ai`. Auto-selected when `NEARAI_API_KEY` is set (or explicitly via `NEARAI_API_MODE=chat_completions`). Tool messages are flattened to plain text for compatibility. Configure with `NEARAI_API_KEY` and `NEARAI_BASE_URL` (default: `https://cloud-api.near.ai`).
**OpenAI-compatible** -- Any endpoint that speaks the OpenAI API (vLLM, LiteLLM, OpenRouter, etc.). Configure with `LLM_BASE_URL`, `LLM_API_KEY` (optional), `LLM_MODEL`. Set `LLM_EXTRA_HEADERS` to inject custom HTTP headers into every request (format: `Key:Value,Key2:Value2`), useful for OpenRouter attribution headers like `HTTP-Referer` and `X-Title`.
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API. Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
## Database
Dual-backend persistence (PostgreSQL + libSQL/Turso). **All new persistence features must support both backends** — see [src/db/CLAUDE.md](src/db/CLAUDE.md) for schema, SQL dialect differences, adding operations, and libSQL limitations.
IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable.
**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL).
### Backends
| Backend | Feature Flag | Default | Use Case |
|---------|-------------|---------|----------|
| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments |
| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud |
Implement every new operation in both `src/db/postgres.rs` and `src/db/libsql/mod.rs`. Test in isolation:
```bash
cargo check # postgres (default)
cargo check --no-default-features --features libsql # libsql only
cargo check --all-features # both
# Build with PostgreSQL only (default)
cargo build
# Build with libSQL only
cargo build --no-default-features --features libsql
# Build with both backends available
cargo build --features "postgres,libsql"
```
### Database Trait
The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence:
- Conversations, messages, metadata
- Jobs, actions, LLM calls, estimation snapshots
- Sandbox jobs, job events
- Routines, routine runs
- Tool failures, settings
- Workspace: documents, chunks, hybrid search
Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL.
### Schema
**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`.
**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types:
- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT`
- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx`
- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers
- PL/pgSQL functions -> SQLite triggers
**Tables (both backends):**
**Core:**
- `conversations` - Multi-channel conversation tracking
- `agent_jobs` - Job metadata and status
- `job_actions` - Event-sourced tool executions
- `dynamic_tools` - Agent-built tools
- `llm_calls` - Cost tracking
- `estimation_snapshots` - Learning data
**Workspace/Memory:**
- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md")
- `memory_chunks` - Chunked content with FTS and vector indexes
- `heartbeat_state` - Periodic execution tracking
**Other:**
- `routines`, `routine_runs` - Scheduled/reactive execution
- `settings` - Per-user key-value settings
- `tool_failures` - Self-repair tracking
- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure
Database configuration: see Configuration section above.
### Current Limitations (libSQL backend)
- **Workspace/memory system** not yet wired through Database trait (requires Store migration)
- **Secrets store** not yet available (still requires PostgresSecretsStore)
- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented)
- **Settings reload from DB** skipped (Config::from_db requires Store)
- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet)
- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage.
- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields.
## Safety Layer
All external tool output passes through `SafetyLayer`:
@@ -653,8 +625,8 @@ Key test patterns:
4. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed)
5. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access
6. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools
7. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard
8. **Observability backends** - Only `log` and `noop` implemented; OpenTelemetry/Prometheus not yet supported
7. **Webhook trigger endpoint** - Routines webhook trigger not yet exposed in web gateway
8. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard
## Tool Architecture
@@ -668,8 +640,8 @@ See `src/tools/README.md` for full tool architecture, adding new tools (built-in
1. Create `src/channels/my_channel.rs`
2. Implement the `Channel` trait
3. Add config in `src/config/channels.rs`
4. Wire up in `src/app.rs` channel setup section
3. Add config in `src/config.rs`
4. Wire up in `main.rs` channel setup section
## Debugging
@@ -701,11 +673,6 @@ for that module's behavior. When modifying code in a module that has a spec:
| `src/setup/` | `src/setup/README.md` |
| `src/workspace/` | `src/workspace/README.md` |
| `src/tools/` | `src/tools/README.md` |
| `src/agent/` | `src/agent/CLAUDE.md` |
| `src/channels/web/` | `src/channels/web/CLAUDE.md` |
| `src/db/` | `src/db/CLAUDE.md` |
| `src/llm/` | `src/llm/CLAUDE.md` |
| `tests/e2e/` | `tests/e2e/CLAUDE.md` |
## Workspace & Memory System
-862
View File
@@ -1,862 +0,0 @@
# IronClaw Coverage Plan: 63.3% to 95%
> Generated 2025-03-06 from [Codecov](https://app.codecov.io/gh/nearai/ironclaw/tree/main/src)
## Current State
| Metric | Value |
|--------|-------|
| **Current coverage** | 48,571 / 76,694 lines = **63.33%** |
| **Target** | 72,859 / 76,694 lines = **95.0%** |
| **Gap** | **24,288 lines** need coverage |
| **Files >= 95%** | 43 / 239 |
| **Files < 95%** | 196 (27,872 total misses) |
## Module Summary
Sorted by uncovered lines (descending):
| Module | Lines | Hits | Miss | Coverage | Priority |
|--------|------:|-----:|-----:|---------:|----------|
| `channels/` | 14,079 | 8,677 | 5,402 | 61.6% | P0 |
| `tools/` | 13,445 | 9,407 | 4,038 | 70.0% | P1 |
| `agent/` | 9,152 | 6,096 | 3,056 | 66.6% | P0 |
| `setup/` | 3,005 | 462 | 2,543 | 15.4% | P1 |
| `extensions/` | 3,540 | 1,298 | 2,242 | 36.7% | P0 |
| `cli/` | 2,834 | 697 | 2,137 | 24.6% | P1 |
| `history/` | 1,626 | 0 | 1,626 | 0.0% | P0 |
| `llm/` | 7,029 | 5,776 | 1,253 | 82.2% | P2 |
| `(root)` | 4,122 | 3,121 | 1,001 | 75.7% | P2 |
| `worker/` | 1,274 | 480 | 794 | 37.7% | P1 |
| `sandbox/` | 1,615 | 897 | 718 | 55.5% | P2 |
| `registry/` | 1,588 | 1,107 | 481 | 69.7% | P2 |
| `db/` | 921 | 441 | 480 | 47.9% | P1 |
| `workspace/` | 2,006 | 1,584 | 422 | 79.0% | P2 |
| `orchestrator/` | 1,199 | 795 | 404 | 66.3% | P2 |
| `config/` | 1,464 | 1,095 | 369 | 74.8% | P2 |
| `hooks/` | 1,379 | 1,081 | 298 | 78.4% | P2 |
| `secrets/` | 687 | 407 | 280 | 59.2% | P2 |
| `skills/` | 1,714 | 1,585 | 129 | 92.5% | P3 |
| `context/` | 693 | 586 | 107 | 84.6% | P3 |
| `estimation/` | 467 | 369 | 98 | 79.0% | P3 |
| `safety/` | 1,424 | 1,337 | 87 | 93.9% | P3 |
| `evaluation/` | 226 | 152 | 74 | 67.3% | P3 |
| `pairing/` | 498 | 446 | 52 | 89.6% | P3 |
| `tunnel/` | 391 | 368 | 23 | 94.1% | P3 |
| `observability/` | 316 | 307 | 9 | 97.2% | Done |
## Top 40 Files by Uncovered Lines
These files account for the vast majority of the coverage gap:
| File | Lines | Miss | Coverage | Lines to 95% |
|------|------:|-----:|---------:|--------------:|
| `src/extensions/manager.rs` | 2,404 | 2,083 | 13.3% | 1,962 |
| `src/setup/wizard.rs` | 2,150 | 1,789 | 16.8% | 1,681 |
| `src/history/store.rs` | 1,486 | 1,486 | 0.0% | 1,411 |
| `src/channels/web/server.rs` | 1,985 | 993 | 50.0% | 893 |
| `src/channels/wasm/wrapper.rs` | 2,237 | 934 | 58.2% | 822 |
| `src/agent/thread_ops.rs` | 1,044 | 763 | 26.9% | 710 |
| `src/cli/tool.rs` | 757 | 735 | 2.9% | 697 |
| `src/setup/channels.rs` | 645 | 596 | 7.6% | 563 |
| `src/agent/commands.rs` | 587 | 587 | 0.0% | 557 |
| `src/main.rs` | 740 | 522 | 29.4% | 485 |
| `src/channels/web/handlers/jobs.rs` | 513 | 456 | 11.1% | 430 |
| `src/tools/builder/core.rs` | 524 | 456 | 13.0% | 429 |
| `src/agent/worker.rs` | 1,078 | 467 | 56.7% | 413 |
| `src/channels/web/handlers/chat.rs` | 564 | 417 | 26.1% | 388 |
| `src/tools/wasm/wrapper.rs` | 1,005 | 436 | 56.6% | 385 |
| `src/channels/signal.rs` | 1,814 | 472 | 74.0% | 381 |
| `src/tools/mcp/auth.rs` | 472 | 378 | 19.9% | 354 |
| `src/worker/runtime.rs` | 350 | 330 | 5.7% | 312 |
| `src/tools/builtin/job.rs` | 1,014 | 359 | 64.6% | 308 |
| `src/cli/mcp.rs` | 322 | 319 | 0.9% | 302 |
| `src/cli/oauth_defaults.rs` | 730 | 335 | 54.1% | 298 |
| `src/llm/nearai_chat.rs` | 854 | 340 | 60.2% | 297 |
| `src/sandbox/container.rs` | 407 | 317 | 22.1% | 296 |
| `src/tools/mcp/client.rs` | 341 | 291 | 14.7% | 273 |
| `src/registry/installer.rs` | 765 | 311 | 59.3% | 272 |
| `src/orchestrator/job_manager.rs` | 405 | 270 | 33.3% | 249 |
| `src/channels/web/handlers/routines.rs` | 249 | 249 | 0.0% | 236 |
| `src/agent/scheduler.rs` | 559 | 263 | 53.0% | 235 |
| `src/tools/wasm/storage.rs` | 296 | 243 | 17.9% | 228 |
| `src/channels/repl.rs` | 233 | 233 | 0.0% | 221 |
| `src/llm/session.rs` | 413 | 242 | 41.4% | 221 |
| `src/worker/claude_bridge.rs` | 629 | 247 | 60.7% | 215 |
| `src/agent/agent_loop.rs` | 523 | 234 | 55.2% | 207 |
| `src/worker/api.rs` | 258 | 207 | 19.8% | 194 |
| `src/sandbox/proxy/http.rs` | 307 | 192 | 37.5% | 176 |
| `src/channels/wasm/storage.rs` | 182 | 182 | 0.0% | 172 |
| `src/cli/registry.rs` | 177 | 177 | 0.0% | 168 |
| `src/llm/reasoning.rs` | 1,163 | 219 | 81.2% | 160 |
| `src/tools/builder/testing.rs` | 308 | 174 | 43.5% | 158 |
| `src/db/postgres.rs` | 166 | 166 | 0.0% | 157 |
---
## Tier 1 -- High-Impact Unit Tests (~8,500 lines)
Pure logic, serialization, and database queries testable in isolation without real
infrastructure. Highest coverage gain per unit of effort.
### `src/history/store.rs` -- 0% -> 95% (+1,411 lines)
PostgreSQL repository layer (conversations, jobs, actions, LLM calls, estimation
snapshots). Test query construction and result mapping. Can use the libSQL backend
as a real in-memory database or test doubles for the `Database` trait.
**Tests to write:**
- `test_store_conversation_crud` -- create, read, update, delete conversations
- `test_store_job_lifecycle` -- insert job, update status through state machine
- `test_store_action_recording` -- record and query job actions
- `test_store_llm_call_tracking` -- insert and aggregate LLM call records
- `test_store_estimation_snapshots` -- save and retrieve estimation data
### `src/history/analytics.rs` -- 0% -> 95% (+133 lines)
Aggregation queries (JobStats, ToolStats). Test the query builders and result
deserialization.
**Tests to write:**
- `test_job_stats_aggregation` -- verify counts, durations, success rates
- `test_tool_stats_ranking` -- verify tool usage frequency sorting
- `test_analytics_empty_db` -- graceful handling of no data
### `src/extensions/manager.rs` -- 13.3% -> 95% (+1,962 lines)
Largest single file gap. Extension lifecycle orchestration (install, auth,
activate, remove), config parsing, and state transitions.
**Tests to write:**
- `test_extension_install_from_manifest` -- parse manifest, create extension record
- `test_extension_auth_flow` -- OAuth token setup, credential storage
- `test_extension_activate_deactivate` -- state transitions, tool registration
- `test_extension_remove_cleanup` -- remove extension, clean up artifacts
- `test_extension_config_validation` -- reject invalid configs, handle defaults
- `test_extension_list_filtering` -- filter by status, type, search query
- `test_extension_capability_check` -- verify required capabilities before activation
### `src/extensions/discovery.rs` -- 27.8% -> 95% (+125 lines)
Extension discovery from filesystem and registry.
**Tests to write:**
- `test_discover_local_extensions` -- scan directory, parse manifests
- `test_discover_skip_invalid` -- gracefully skip malformed extension dirs
- `test_discover_dedup` -- handle duplicate extensions across paths
### `src/tools/builder/core.rs` -- 13% -> 95% (+429 lines)
`BuildRequirement`, `SoftwareType`, `Language` types and project scaffolding.
**Tests to write:**
- `test_build_requirement_parsing` -- deserialize from JSON
- `test_scaffold_project_structure` -- verify generated file tree
- `test_language_detection` -- detect language from file extensions
- `test_software_type_constraints` -- validate type-specific requirements
### `src/tools/builder/testing.rs` -- 43.5% -> 95% (+158 lines)
Test harness integration for built tools.
**Tests to write:**
- `test_harness_setup_teardown` -- lifecycle of test environment
- `test_harness_run_tests` -- execute tests and capture results
- `test_harness_failure_reporting` -- verify error details on test failure
### `src/tools/mcp/auth.rs` -- 19.9% -> 95% (+354 lines)
OAuth token management for MCP servers.
**Tests to write:**
- `test_token_refresh_on_expiry` -- auto-refresh when token expires
- `test_token_header_injection` -- correct Authorization header format
- `test_token_persistence` -- save/load tokens across restarts
- `test_oauth_pkce_flow` -- code verifier/challenge generation
- `test_auth_config_parsing` -- parse various auth config formats
### `src/tools/mcp/client.rs` -- 14.7% -> 95% (+273 lines)
JSON-RPC client for MCP protocol.
**Tests to write:**
- `test_jsonrpc_request_serialization` -- correct JSON-RPC 2.0 format
- `test_jsonrpc_response_parsing` -- handle success, error, and batch responses
- `test_jsonrpc_error_codes` -- map MCP error codes to ToolError
- `test_tool_list_discovery` -- parse tools/list response
- `test_tool_call_roundtrip` -- serialize call, parse result
### `src/tools/wasm/storage.rs` -- 17.9% -> 95% (+228 lines)
WASM tool persistence (store, load, delete, list).
**Tests to write:**
- `test_wasm_tool_store_roundtrip` -- store and retrieve tool binary + metadata
- `test_wasm_tool_delete` -- remove tool and verify gone
- `test_wasm_tool_list_filtering` -- filter by name, capability
- `test_wasm_tool_update_metadata` -- update without re-uploading binary
### `src/tools/wasm/wrapper.rs` -- 56.6% -> 95% (+385 lines)
Tool trait wrapper for WASM modules.
**Tests to write:**
- `test_wasm_param_marshalling` -- JSON params to WASM component model types
- `test_wasm_output_conversion` -- WASM return values to ToolOutput
- `test_wasm_error_propagation` -- WASM traps to ToolError
- `test_wasm_fuel_exhaustion` -- verify fuel limit enforcement
- `test_wasm_memory_limit` -- verify memory ceiling
### `src/tools/wasm/loader.rs` -- 62.4% -> 95% (+156 lines)
WASM tool discovery from filesystem.
**Tests to write:**
- `test_loader_scan_directory` -- find .wasm files with capabilities.json
- `test_loader_skip_invalid` -- skip files without valid WIT exports
- `test_loader_cache_invalidation` -- reload when file changes
### `src/tools/builtin/job.rs` -- 64.6% -> 95% (+308 lines)
Job management tools (CreateJob, ListJobs, JobStatus, CancelJob).
**Tests to write:**
- `test_create_job_params` -- validate required/optional parameters
- `test_list_jobs_formatting` -- verify output structure
- `test_job_status_transitions` -- query status at each state
- `test_cancel_job_running` -- cancel an in-progress job
- `test_cancel_job_completed` -- error on already-completed job
### `src/secrets/store.rs` -- 48.1% -> 95% (+145 lines)
Encrypted secret storage.
**Tests to write:**
- `test_secret_store_roundtrip` -- store encrypted, retrieve decrypted
- `test_secret_update` -- overwrite existing secret
- `test_secret_delete` -- remove and verify inaccessible
- `test_secret_list_redacted` -- list shows names but not values
### `src/llm/session.rs` -- 41.4% -> 95% (+221 lines)
Session token management with auto-renewal.
**Tests to write:**
- `test_session_token_parsing` -- parse `sess_xxx` format
- `test_session_expiry_detection` -- detect expired tokens
- `test_session_auto_renewal` -- trigger renewal before expiry
- `test_session_concurrent_renewal` -- only one renewal in flight
### `src/llm/nearai_chat.rs` -- 60.2% -> 95% (+297 lines)
NEAR AI Chat Completions provider.
**Tests to write:**
- `test_nearai_request_building` -- correct endpoint, headers, body
- `test_nearai_response_parsing` -- parse streaming and non-streaming responses
- `test_nearai_tool_message_flattening` -- tool messages flattened to text
- `test_nearai_auth_modes` -- session token vs API key auth
- `test_nearai_error_handling` -- rate limits, auth failures, server errors
### `src/llm/mod.rs` -- 53.7% -> 95% (+112 lines)
Provider factory and backend selection.
**Tests to write:**
- `test_provider_factory_nearai` -- select NEAR AI from config
- `test_provider_factory_openai` -- select OpenAI from config
- `test_provider_factory_ollama` -- select Ollama from config
- `test_provider_factory_invalid` -- error on unknown backend
### `src/llm/reasoning.rs` -- 81.2% -> 95% (+160 lines)
Planning, tool selection, evaluation logic.
**Tests to write:**
- `test_reasoning_step_parsing` -- parse planning steps from LLM output
- `test_tool_selection_scoring` -- rank tools by relevance
- `test_evaluation_rubric` -- score completions against criteria
- `test_reasoning_with_no_tools` -- handle tool-less responses
### `src/db/postgres.rs` -- 0% -> 95% (+157 lines)
PostgreSQL backend delegation to Store + Repository.
**Tests to write:**
- `test_postgres_backend_delegates` -- verify delegation pattern (trait-level)
- `test_postgres_connection_config` -- TLS, pool size, timeout parsing
### `src/workspace/mod.rs` -- 75.9% -> 95% (+109 lines)
Memory operations (write, read, search, tree).
**Tests to write:**
- `test_workspace_write_read` -- write document, read it back
- `test_workspace_search_hybrid` -- FTS + vector search via RRF
- `test_workspace_tree` -- directory listing of memory filesystem
- `test_workspace_overwrite` -- update existing document
### `src/workspace/embeddings.rs` -- 35.1% -> 95% (~100 lines)
Embedding provider abstraction.
**Tests to write:**
- `test_embedding_dimension_handling` -- verify dimension config
- `test_embedding_batch_processing` -- batch multiple chunks
- `test_embedding_provider_fallback` -- graceful degradation when unavailable
---
## Tier 2 -- Trace Tests (~7,000 lines)
End-to-end tests that exercise the agent loop, worker, scheduler, and dispatcher
by replaying LLM traces through `TestRig` (see `tests/support/test_rig.rs`). Each
trace test covers multiple modules simultaneously, making them high-leverage.
Each trace test needs:
1. A JSON fixture in `tests/fixtures/llm_traces/`
2. A test file in `tests/` using `TestRigBuilder`
### Trace: Thread Operations
**Covers:** `agent/thread_ops.rs` (+710 lines)
Test thread creation, listing, switching, and deletion via trace replay.
**Fixture:** `thread_operations.json`
**Tests:**
- `test_thread_create_and_switch` -- create thread, switch to it, verify context
- `test_thread_list` -- list all threads, verify metadata
- `test_thread_delete` -- delete thread, verify removal
- `test_thread_switch_nonexistent` -- error handling for missing thread
### Trace: Agent Commands
**Covers:** `agent/commands.rs` (+557 lines)
Test slash commands through the agent loop.
**Fixture:** `agent_commands.json`
**Tests:**
- `test_command_help` -- /help returns command list
- `test_command_clear` -- /clear resets conversation
- `test_command_compact` -- /compact triggers summarization
- `test_command_undo_redo` -- /undo then /redo restores state
- `test_command_status` -- /status shows agent state
### Trace: Worker Multi-Turn Execution
**Covers:** `agent/worker.rs` (+413 lines), `agent/agent_loop.rs` (+207 lines)
Test multi-turn tool calling, error recovery, and completion flows.
**Fixture:** `worker_multi_turn.json`
**Tests:**
- `test_worker_sequential_tools` -- call tool A, then tool B based on A's result
- `test_worker_tool_error_recovery` -- tool fails, agent retries or adapts
- `test_worker_max_turns` -- verify turn limit enforcement
### Trace: Scheduler Parallel Jobs
**Covers:** `agent/scheduler.rs` (+235 lines)
Test parallel job dispatch and completion tracking.
**Fixture:** `scheduler_parallel.json`
**Tests:**
- `test_scheduler_parallel_dispatch` -- dispatch 3 jobs, all complete
- `test_scheduler_job_dependency` -- job B waits for job A
- `test_scheduler_stuck_detection` -- detect and recover stuck job
### Trace: Dispatcher Skill Selection
**Covers:** `agent/dispatcher.rs` (+153 lines)
Test skill-aware routing and tool attenuation.
**Fixture:** `dispatcher_skills.json`
**Tests:**
- `test_dispatcher_skill_match` -- match message to skill, inject prompt
- `test_dispatcher_tool_attenuation` -- installed skill loses dangerous tools
- `test_dispatcher_no_skill` -- fallback when no skill matches
### Trace: Routine Execution
**Covers:** `agent/routine_engine.rs` (~80 lines), `agent/routine.rs` (~40 lines)
Test cron tick and event-triggered routine execution.
**Fixture:** `routine_execution.json`
**Tests:**
- `test_routine_cron_trigger` -- routine fires on schedule
- `test_routine_event_trigger` -- routine fires on matching event
- `test_routine_guardrails` -- routine respects policy constraints
### Trace: Compaction and Context Pressure
**Covers:** `agent/compaction.rs` (~50 lines), `agent/context_monitor.rs` (~30 lines)
Test turn summarization and memory pressure detection.
**Fixture:** `compaction_flow.json`
**Tests:**
- `test_compaction_triggers_at_threshold` -- summarize when context exceeds limit
- `test_compaction_preserves_recent` -- keep recent turns intact
- `test_context_pressure_warning` -- emit warning at high usage
### Trace: Job Tool Coverage
**Covers:** `tools/builtin/job.rs` (+308 lines), `tools/builtin/skill_tools.rs` (+110 lines)
Test job and skill management tools through agent execution.
**Fixture:** `job_and_skill_tools.json`
**Tests:**
- `test_create_and_list_jobs` -- create job, list shows it
- `test_job_status_query` -- query status of running job
- `test_skill_list_and_search` -- list local skills, search registry
### Trace: Memory Tools
**Covers:** `tools/builtin/memory.rs` (~20 lines), `workspace/` (+109 lines)
Test memory operations through agent tool calls.
**Fixture:** `memory_tools.json`
**Tests:**
- `test_memory_write_and_search` -- write doc, search finds it
- `test_memory_read_by_path` -- read specific document
- `test_memory_tree` -- list memory filesystem structure
### Trace: Extension Management
**Covers:** `tools/builtin/extension_tools.rs` (~40 lines)
Test extension lifecycle via agent tool calls.
**Fixture:** `extension_management.json`
**Tests:**
- `test_extension_install_via_tool` -- agent installs an extension
- `test_extension_auth_via_tool` -- agent configures auth
- `test_extension_activate_via_tool` -- agent activates extension
### Trace: Self-Repair
**Covers:** `agent/self_repair.rs` (~40 lines)
Test stuck job detection and recovery.
**Fixture:** `self_repair.json`
**Tests:**
- `test_stuck_job_detected` -- job stuck for > threshold triggers repair
- `test_stuck_job_recovered` -- recovery restarts job successfully
- `test_stuck_job_fails_permanently` -- recovery fails, job marked failed
### Trace: Heartbeat
**Covers:** `agent/heartbeat.rs` (+80 lines)
Test periodic proactive execution.
**Fixture:** `heartbeat.json`
**Tests:**
- `test_heartbeat_periodic_fire` -- heartbeat triggers at interval
- `test_heartbeat_reads_checklist` -- reads HEARTBEAT.md, processes items
- `test_heartbeat_notification` -- sends notification on findings
---
## Tier 3 -- Web/Channel Handler Tests (~4,500 lines)
Test HTTP handlers and SSE/WS endpoints using `axum_test` or
`tower::ServiceExt::oneshot` with a real router and in-memory database.
### `src/channels/web/server.rs` -- 50% -> 95% (+893 lines)
The single biggest web gap. 40+ API endpoints.
**Tests to write:**
- `test_api_health` -- GET /health returns 200
- `test_api_chat_submit` -- POST /api/chat sends message
- `test_api_jobs_list` -- GET /api/jobs returns job list
- `test_api_jobs_create` -- POST /api/jobs creates job
- `test_api_routines_crud` -- full CRUD cycle for routines
- `test_api_settings_get_set` -- GET/PUT settings
- `test_api_memory_search` -- POST /api/memory/search
- `test_api_extensions_list` -- GET /api/extensions
- `test_api_skills_list` -- GET /api/skills
- `test_api_sse_connect` -- SSE stream connects and receives events
- `test_api_auth_required` -- endpoints reject missing/bad tokens
- `test_api_cors_headers` -- verify CORS configuration
### `src/channels/web/handlers/chat.rs` -- 26.1% -> 95% (+388 lines)
Chat message submission and SSE streaming.
**Tests to write:**
- `test_chat_submit_message` -- submit message, receive response
- `test_chat_sse_stream` -- verify SSE event format
- `test_chat_thread_context` -- messages scoped to thread
- `test_chat_invalid_payload` -- reject malformed requests
### `src/channels/web/handlers/jobs.rs` -- 11.1% -> 95% (+430 lines)
Job CRUD endpoints.
**Tests to write:**
- `test_jobs_list_empty` -- empty list returns []
- `test_jobs_create_and_get` -- create, then GET by ID
- `test_jobs_cancel` -- cancel running job
- `test_jobs_filter_by_status` -- filter by pending/running/completed
- `test_jobs_pagination` -- limit/offset parameters
### `src/channels/web/handlers/routines.rs` -- 0% -> 95% (+236 lines)
Routine CRUD endpoints.
**Tests to write:**
- `test_routines_create` -- POST creates routine
- `test_routines_list` -- GET lists all routines
- `test_routines_update` -- PUT updates routine config
- `test_routines_delete` -- DELETE removes routine
- `test_routines_history` -- GET history for a routine
### `src/channels/web/handlers/extensions.rs` -- 0% -> 95% (+129 lines)
Extension management endpoints.
**Tests to write:**
- `test_extensions_list` -- list installed extensions
- `test_extensions_install` -- install from manifest URL
- `test_extensions_activate` -- activate/deactivate toggle
- `test_extensions_remove` -- remove installed extension
### `src/channels/web/handlers/memory.rs` -- 0% -> 95% (+110 lines)
Memory/workspace endpoints.
**Tests to write:**
- `test_memory_search` -- search returns ranked results
- `test_memory_write` -- write a document
- `test_memory_read` -- read by path
- `test_memory_tree` -- tree returns filesystem structure
### `src/channels/web/handlers/settings.rs` -- 0% -> 95% (+103 lines)
Settings endpoints.
**Tests to write:**
- `test_settings_get` -- retrieve current settings
- `test_settings_update` -- update individual setting
- `test_settings_validation` -- reject invalid setting values
### `src/channels/web/handlers/static_files.rs` -- 0% -> 95% (+97 lines)
Static file serving.
**Tests to write:**
- `test_static_index_html` -- GET / serves index.html
- `test_static_css_js` -- serve CSS/JS with correct content types
- `test_static_404` -- missing file returns 404
### `src/channels/wasm/wrapper.rs` -- 58.2% -> 95% (+822 lines)
WASM channel wrapper (message routing, lifecycle).
**Tests to write:**
- `test_wasm_channel_start` -- initialize WASM channel module
- `test_wasm_channel_message_routing` -- route incoming message to WASM
- `test_wasm_channel_response` -- return WASM response to caller
- `test_wasm_channel_error_handling` -- handle WASM trap gracefully
- `test_wasm_channel_lifecycle` -- start, process, shutdown
### `src/channels/wasm/loader.rs` -- 38.1% -> 95% (+141 lines)
WASM channel discovery.
**Tests to write:**
- `test_channel_loader_scan` -- find channel WASM modules
- `test_channel_loader_validation` -- reject invalid modules
- `test_channel_loader_manifest` -- parse channel capabilities
### `src/channels/wasm/storage.rs` -- 0% -> 95% (+172 lines)
WASM channel state persistence.
**Tests to write:**
- `test_channel_storage_save_load` -- persist and restore channel state
- `test_channel_storage_isolation` -- per-channel state isolation
- `test_channel_storage_cleanup` -- remove state on channel uninstall
### `src/channels/signal.rs` -- 74% -> 95% (+381 lines)
Signal protocol channel.
**Tests to write:**
- `test_signal_message_send` -- send encrypted message
- `test_signal_message_receive` -- decrypt incoming message
- `test_signal_attachment_handling` -- handle media attachments
- `test_signal_group_message` -- group chat routing
- `test_signal_error_handling` -- handle connection failures
### `src/channels/repl.rs` -- 0% -> 95% (+221 lines)
Simple REPL channel.
**Tests to write:**
- `test_repl_input_parsing` -- parse user input lines
- `test_repl_output_formatting` -- format agent responses
- `test_repl_multiline` -- handle multi-line input
- `test_repl_special_commands` -- handle /quit, /help
---
## Tier 4 -- CLI Tests (~2,100 lines)
CLI subcommands can be tested by invoking clap-parsed command structs directly
or by calling the handler functions with constructed arguments.
### `src/cli/tool.rs` -- 2.9% -> 95% (+697 lines)
Tool CLI (install, list, remove, build).
**Tests to write:**
- `test_cli_tool_list` -- list installed tools
- `test_cli_tool_install_local` -- install from local .wasm file
- `test_cli_tool_install_registry` -- install from registry
- `test_cli_tool_remove` -- remove installed tool
- `test_cli_tool_build` -- scaffold and build tool project
- `test_cli_tool_info` -- display tool details
### `src/cli/mcp.rs` -- 0.9% -> 95% (+302 lines)
MCP server management CLI.
**Tests to write:**
- `test_cli_mcp_list` -- list configured MCP servers
- `test_cli_mcp_add` -- add MCP server config
- `test_cli_mcp_remove` -- remove MCP server config
- `test_cli_mcp_tools` -- list tools from MCP server
- `test_cli_mcp_test_connection` -- verify MCP server reachable
### `src/cli/oauth_defaults.rs` -- 54.1% -> 95% (+298 lines)
OAuth default configurations.
**Tests to write:**
- `test_oauth_defaults_loading` -- load default OAuth configs
- `test_oauth_url_construction` -- build auth/token URLs
- `test_oauth_scope_merging` -- merge requested scopes with defaults
- `test_oauth_provider_lookup` -- lookup by provider name
### `src/cli/registry.rs` -- 0% -> 95% (+168 lines)
Registry CLI commands.
**Tests to write:**
- `test_cli_registry_search` -- search for packages
- `test_cli_registry_install` -- install package from registry
- `test_cli_registry_info` -- display package details
### `src/cli/status.rs` -- 0% -> 95% (+142 lines)
Status display commands.
**Tests to write:**
- `test_cli_status_gathering` -- collect system status info
- `test_cli_status_formatting` -- render status output
- `test_cli_status_components` -- check individual components
### `src/cli/memory.rs` -- 15.5% -> 95% (+138 lines)
Memory CLI subcommands.
**Tests to write:**
- `test_cli_memory_search` -- search workspace from CLI
- `test_cli_memory_write` -- write document from CLI
- `test_cli_memory_read` -- read document from CLI
- `test_cli_memory_tree` -- display memory tree
### `src/cli/doctor.rs` -- 28.7% -> 95% (+115 lines)
Diagnostic checks.
**Tests to write:**
- `test_doctor_check_database` -- verify DB connectivity check
- `test_doctor_check_llm` -- verify LLM provider check
- `test_doctor_check_tools` -- verify tool availability check
- `test_doctor_report_format` -- verify output format
### `src/cli/config.rs` -- 36.5% -> 95% (~100 lines)
Config CLI subcommands.
**Tests to write:**
- `test_cli_config_get` -- read config value
- `test_cli_config_set` -- write config value
- `test_cli_config_list` -- list all config keys
- `test_cli_config_reset` -- reset to defaults
---
## Tier 5 -- Setup/Infra Tests (~2,400 lines)
Hardest to test: interactive wizards, Docker, process spawning. Strategy: extract
pure logic into testable functions, test the interactive parts by injecting mock
input.
### `src/setup/wizard.rs` -- 16.8% -> 95% (+1,681 lines)
7-step interactive onboarding wizard. Refactor to extract validation functions,
step logic, and config generation into testable units.
**Tests to write:**
- `test_wizard_step_validation` -- each step validates input correctly
- `test_wizard_config_generation` -- generate config from wizard answers
- `test_wizard_default_values` -- verify sensible defaults
- `test_wizard_skip_completed` -- skip already-configured steps
- `test_wizard_llm_backend_selection` -- provider-specific config paths
- `test_wizard_channel_setup` -- channel configuration logic
### `src/setup/channels.rs` -- 7.6% -> 95% (+563 lines)
Channel setup helpers.
**Tests to write:**
- `test_channel_setup_defaults` -- default channel configuration
- `test_channel_setup_validation` -- reject invalid channel configs
- `test_channel_setup_telegram` -- Telegram-specific setup logic
- `test_channel_setup_signal` -- Signal-specific setup logic
- `test_channel_setup_webhook` -- webhook URL validation
### `src/setup/prompts.rs` -- 24.8% -> 95% (+147 lines)
Terminal prompt utilities.
**Tests to write:**
- `test_prompt_select` -- selection from list
- `test_prompt_confirm` -- yes/no confirmation
- `test_prompt_secret` -- masked input
- `test_prompt_validation` -- input validation rules
### `src/sandbox/container.rs` -- 22.1% -> 95% (+296 lines)
Docker container lifecycle. Test command construction without actual Docker.
**Tests to write:**
- `test_container_config_to_docker_args` -- generate correct docker run args
- `test_container_volume_mounts` -- workspace mount configuration
- `test_container_env_scrubbing` -- sensitive env vars removed
- `test_container_resource_limits` -- CPU/memory limit args
- `test_container_network_config` -- proxy network setup
### `src/sandbox/manager.rs` -- 59% -> 95% (+114 lines)
Sandbox orchestration.
**Tests to write:**
- `test_sandbox_policy_enforcement` -- policy to container config mapping
- `test_sandbox_cleanup` -- cleanup on job completion
- `test_sandbox_concurrent_limit` -- enforce max concurrent containers
### `src/sandbox/proxy/http.rs` -- 37.5% -> 95% (+176 lines)
HTTP proxy for container network access.
**Tests to write:**
- `test_proxy_allowlist_enforcement` -- block disallowed domains
- `test_proxy_credential_injection` -- inject auth headers
- `test_proxy_connect_tunnel` -- HTTPS CONNECT method handling
- `test_proxy_logging` -- request/response logging
### `src/worker/runtime.rs` -- 5.7% -> 95% (+312 lines)
Worker execution loop (runs inside containers).
**Tests to write:**
- `test_worker_tool_dispatch` -- dispatch tool call, return result
- `test_worker_llm_interaction` -- send prompt, receive response
- `test_worker_turn_limit` -- enforce max turns
- `test_worker_error_propagation` -- tool error surfaces to agent
### `src/worker/claude_bridge.rs` -- 60.7% -> 95% (+215 lines)
Claude CLI bridge.
**Tests to write:**
- `test_claude_command_construction` -- build claude CLI command
- `test_claude_output_parsing` -- parse claude CLI JSON output
- `test_claude_error_handling` -- handle CLI crashes gracefully
- `test_claude_config_injection` -- inject config dir and model
### `src/worker/api.rs` -- 19.8% -> 95% (+194 lines)
Worker HTTP client to orchestrator.
**Tests to write:**
- `test_worker_api_request_building` -- correct endpoint URLs and headers
- `test_worker_api_response_parsing` -- parse orchestrator responses
- `test_worker_api_auth_token` -- bearer token injection
- `test_worker_api_retry` -- retry on transient failures
### `src/main.rs` -- 29.4% -> 95% (+485 lines)
Entry point and startup. Extract startup logic into testable functions.
**Tests to write:**
- `test_cli_arg_parsing` -- verify clap argument parsing
- `test_startup_config_loading` -- config from env + file
- `test_startup_channel_selection` -- select channels from config
- `test_startup_feature_flags` -- feature-gated code paths
---
## Tier 6 -- Remaining Files to 95% (~2,000 lines)
Smaller files that each need a handful of additional tests.
| File | Lines Needed | Test Focus |
|------|-------------:|------------|
| `src/tools/builtin/skill_tools.rs` | 110 | skill_list, skill_search, skill_install, skill_remove |
| `src/hooks/bundled.rs` | 115 | bundled hook execution, hook discovery |
| `src/registry/installer.rs` | 272 | package download, verification, installation |
| `src/registry/artifacts.rs` | 72 | artifact packaging, checksums |
| `src/orchestrator/job_manager.rs` | 249 | container lifecycle, job routing |
| `src/orchestrator/api.rs` | 125 | LLM proxy, event dispatch endpoints |
| `src/app.rs` | 137 | AppBuilder configuration, startup sequence |
| `src/service.rs` | 120 | service lifecycle, signal handling |
| `src/config/channels.rs` | 55 | channel config parsing |
| `src/config/sandbox.rs` | 61 | sandbox config parsing |
| `src/config/tunnel.rs` | 43 | tunnel config parsing |
| `src/config/mod.rs` | 63 | config merging, env override |
| `src/config/database.rs` | 38 | database URL parsing |
| `src/evaluation/success.rs` | 34 | success evaluator logic |
| `src/evaluation/metrics.rs` | 40 | metrics collection |
| `src/context/manager.rs` | 57 | concurrent job context isolation |
| `src/context/memory.rs` | 36 | action recording, conversation memory |
---
## Execution Priority
Maximize coverage gain per unit of effort:
| Order | Category | Lines Gained | Effort |
|------:|----------|-------------:|--------|
| 1 | Trace tests (Tier 2) | ~7,000 | Medium (high leverage, each test covers many modules) |
| 2 | Unit tests for 0% files (Tier 1 subset) | ~3,500 | Low (pure logic, no infrastructure) |
| 3 | Web handler tests (Tier 3) | ~4,500 | Medium (axum_test + in-memory DB) |
| 4 | Extension/MCP/WASM unit tests (Tier 1 remainder) | ~3,500 | Medium |
| 5 | CLI subcommand tests (Tier 4) | ~2,100 | Low-Medium |
| 6 | Setup wizard extraction + tests (Tier 5) | ~2,400 | High (requires refactoring) |
| 7 | LLM provider tests (Tier 1 subset) | ~800 | Medium |
| 8 | Remaining small files (Tier 6) | ~2,000 | Low |
## Notes
- All trace tests require `--features libsql` and use `TestRigBuilder` from `tests/support/`
- Web handler tests can use `axum::test` helpers or build the router directly
- CLI tests should call handler functions directly, not shell out to the binary
- Setup wizard tests require extracting pure logic from interactive prompts first
- Sandbox/container tests should verify command construction, not run Docker
- Worker tests can use `TraceLlm` for the LLM provider, same as trace tests
Generated
+24 -861
View File
File diff suppressed because it is too large Load Diff
+4 -20
View File
@@ -12,13 +12,14 @@ exclude = [
"tools-src/google-drive",
"tools-src/google-sheets",
"tools-src/google-slides",
"tools-src/okta",
"tools-src/slack",
"tools-src/telegram",
]
[package]
name = "ironclaw"
version = "0.16.1"
version = "0.13.0"
edition = "2024"
rust-version = "1.92"
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
@@ -40,7 +41,7 @@ tokio-stream = { version = "0.1", features = ["sync"] }
futures = "0.3"
# HTTP client
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
# Serialization
serde = { version = "1", features = ["derive"] }
@@ -56,7 +57,7 @@ rustls = { version = "0.23", optional = true, default-features = false }
rustls-native-certs = { version = "0.8", optional = true }
# Database - libSQL/Turso (optional embedded database)
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication", "remote", "tls"] }
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] }
# Error handling
thiserror = "2"
@@ -73,8 +74,6 @@ toml = "0.8"
# Core types
uuid = { version = "1", features = ["v4", "v5", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
chrono-tz = "0.10"
iana-time-zone = "0.1"
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] }
rust_decimal_macros = "1"
@@ -108,9 +107,6 @@ serde_yml = "0.0.12"
dirs = "6"
fs4 = "0.6"
# Semantic versioning
semver = "1"
# Secrecy for sensitive values
secrecy = { version = "0.10", features = ["serde"] }
@@ -133,7 +129,6 @@ wasmparser = "0.220" # WASM binary parsing for validation
# Cryptography for secrets management
aes-gcm = "0.10"
hkdf = "0.12"
hmac = "0.12"
sha2 = "0.10"
blake3 = "1"
rand = "0.8"
@@ -142,11 +137,6 @@ subtle = "2" # Constant-time comparisons for token validation
# Multi-provider LLM support
rig-core = "0.30"
# AWS Bedrock (native Converse API, opt-in via --features bedrock)
aws-config = { version = "1", features = ["behavior-version-latest"], optional = true }
aws-sdk-bedrockruntime = { version = "1", optional = true }
aws-smithy-types = { version = "1", optional = true }
# Docker sandbox
bollard = "0.18"
@@ -154,10 +144,6 @@ bollard = "0.18"
flate2 = "1"
tar = "0.4"
# Document text extraction
pdf-extract = "0.7"
zip = { version = "2", default-features = false, features = ["deflate"] }
# HTTP proxy for sandboxed network access
hyper = { version = "1.5", features = ["server", "http1", "http2"] }
hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] }
@@ -185,7 +171,6 @@ zbus = "4"
[dev-dependencies]
tokio-test = "0.4"
tracing-test = "0.2"
tokio-tungstenite = "0.26"
testcontainers-modules = { version = "0.11", features = ["postgres"] }
pretty_assertions = "1"
@@ -208,7 +193,6 @@ postgres = [
libsql = ["dep:libsql"]
integration = []
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"]
[[test]]
name = "html_to_markdown"
-1
View File
@@ -28,7 +28,6 @@ COPY migrations/ migrations/
COPY registry/ registry/
COPY channels-src/ channels-src/
COPY wit/ wit/
COPY providers.json providers.json
RUN cargo build --release --bin ironclaw
-57
View File
@@ -1,57 +0,0 @@
# Lightweight test Dockerfile for IronClaw web gateway testing.
#
# Build:
# docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test .
#
# Run (each on a different port):
# docker run --rm -p 3003:3003 ironclaw-test
# docker run --rm -p 3004:3003 ironclaw-test
# docker run --rm -p 3005:3003 ironclaw-test
# Stage 1: Build (libsql only — no PostgreSQL dependency)
FROM rust:1.92-slim-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config libssl-dev cmake gcc g++ \
&& rm -rf /var/lib/apt/lists/* \
&& rustup target add wasm32-wasip2 \
&& cargo install wasm-tools
WORKDIR /app
COPY Cargo.toml Cargo.lock ./
COPY build.rs build.rs
COPY src/ src/
COPY tests/ tests/
COPY migrations/ migrations/
COPY registry/ registry/
COPY channels-src/ channels-src/
COPY wit/ wit/
RUN cargo build --release --no-default-features --features libsql --bin ironclaw
# Stage 2: Runtime
FROM debian:bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates libssl3 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /app/target/release/ironclaw /usr/local/bin/ironclaw
RUN useradd -m -u 1000 -s /bin/bash ironclaw
USER ironclaw
WORKDIR /home/ironclaw
EXPOSE 3003
ENV RUST_LOG=ironclaw=info \
GATEWAY_ENABLED=true \
GATEWAY_HOST=0.0.0.0 \
GATEWAY_PORT=3003 \
GATEWAY_AUTH_TOKEN=test \
DATABASE_BACKEND=libsql \
LIBSQL_PATH=/home/ironclaw/test.db \
SANDBOX_ENABLED=false
ENTRYPOINT ["ironclaw", "--no-onboard"]
+7 -24
View File
@@ -39,7 +39,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| Network modes (loopback/LAN/remote) | ✅ | 🚧 | HTTP only |
| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions, per-request `model` override |
| Canvas hosting | ✅ | ❌ | Agent-driven UI |
| Gateway lock (PID-based) | ✅ | | `fs4` flock-based, acquired in `main.rs` before agent startup |
| Gateway lock (PID-based) | ✅ | | |
| launchd/systemd integration | ✅ | ❌ | |
| Bonjour/mDNS discovery | ✅ | ❌ | |
| Tailscale integration | ✅ | ❌ | |
@@ -119,7 +119,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| Mention-based activation | ✅ | ✅ | bot_username + respond_to_all_group_messages |
| Per-group tool policies | ✅ | ❌ | Allow/deny specific tools |
| Thread isolation | ✅ | ✅ | Separate sessions per thread |
| Per-channel media limits | ✅ | | Attachment type in WIT; max 10 per msg, 20MB total, MIME allowlist |
| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits |
| Typing indicators | ✅ | 🚧 | TUI + Telegram typing/actionable status prompts; richer parity pending |
| Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions |
| Group session priming | ✅ | ❌ | Member roster injected for context |
@@ -215,13 +215,9 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| NEAR AI | ✅ | ✅ | - | Primary provider |
| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy; Opus 4.5, Sonnet 4, Sonnet 4.6 |
| OpenAI | ✅ | 🚧 | - | Via NEAR AI proxy |
| AWS Bedrock | ✅ | ✅ | - | Native Converse API via aws-sdk-bedrockruntime (requires `--features bedrock`) |
| Google Gemini | ✅ | | P3 | Via `gemini` adapter |
| io.net | ✅ | | P3 | Via `ionet` adapter |
| Mistral | ✅ | ✅ | P3 | Via `mistral` adapter |
| Yandex AI Studio | ✅ | ✅ | P3 | Via `yandex` adapter |
| Cloudflare Workers AI | ✅ | ✅ | P3 | Via `cloudflare` adapter |
| NVIDIA API | ✅ | ✅ | P3 | Via `nvidia` adapter and `providers.json` |
| AWS Bedrock | ✅ | ❌ | P3 | |
| Google Gemini | ✅ | | P3 | |
| NVIDIA API | ✅ | | P3 | New provider |
| OpenRouter | ✅ | ✅ | - | Via OpenAI-compatible provider (RigAdapter) |
| Tinfoil | ❌ | ✅ | - | Private inference provider (IronClaw-only) |
| OpenAI-compatible | ❌ | ✅ | - | Generic OpenAI-compatible endpoint (RigAdapter) |
@@ -252,32 +248,19 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
| Feature | OpenClaw | IronClaw | Priority | Notes |
|---------|----------|----------|----------|-------|
| WIT inbound-attachment type | N/A | ✅ | P1 | `inbound-attachment` record in channel-host (id, mime_type, filename, size_bytes, source_url, storage_key, extracted_text) |
| WIT outbound attachment type | N/A | ✅ | P1 | `attachment` record in channel (filename, mime_type, data) on `agent-response` |
| WIT on-broadcast export | N/A | ✅ | P1 | Proactive message sending without prior incoming message |
| IncomingMessage attachments | N/A | ✅ | P1 | `IncomingAttachment` struct on `IncomingMessage`, populated from WASM channels |
| OutgoingResponse attachments | N/A | ✅ | P1 | File paths on `OutgoingResponse`, read from disk and sent as WIT attachments |
| Attachment security (size/MIME) | N/A | ✅ | P1 | Inbound: max 10, 20MB total, MIME allowlist. Outbound: 50MB total |
| Telegram media parsing | ✅ | ✅ | P1 | Photo, document, audio, video, voice, sticker parsed and emitted as attachments |
| Telegram media sending | ✅ | ✅ | P1 | sendPhoto/sendDocument multipart upload, auto photo→document fallback >10MB |
| Slack file parsing | ✅ | ✅ | P1 | `files` array from Events API parsed into attachments |
| WhatsApp media parsing | ✅ | ✅ | P1 | Image, audio, video, document parsed with caption as extracted_text |
| Discord attachment parsing | ✅ | ❌ | P2 | Discord interaction payloads don't include file attachments (needs message events) |
| HTTP tool save_to | N/A | ✅ | P1 | Download binary files to /tmp/ for attachment sending (50MB limit, path traversal protection) |
| Credential env var fallback | N/A | ✅ | P2 | Channels can use env vars (e.g., TELEGRAM_BOT_TOKEN) when secrets store not configured |
| Image processing (Sharp) | ✅ | ❌ | P2 | Resize, format convert |
| Configurable image resize dims | ✅ | ❌ | P2 | Per-agent dimension config |
| Multiple images per tool call | ✅ | ❌ | P2 | Single tool invocation, multiple images |
| Audio transcription | ✅ | ❌ | P2 | |
| Video support | ✅ | ❌ | P3 | |
| PDF parsing | ✅ | ❌ | P2 | pdfjs-dist |
| MIME detection | ✅ | | P2 | MIME allowlist in host validates attachment types |
| MIME detection | ✅ | | P2 | |
| Media caching | ✅ | ❌ | P3 | |
| Vision model integration | ✅ | ❌ | P2 | Image understanding |
| TTS (Edge TTS) | ✅ | ❌ | P3 | Text-to-speech |
| TTS (OpenAI) | ✅ | ❌ | P3 | |
| Incremental TTS playback | ✅ | ❌ | P3 | iOS progressive playback |
| Sticker-to-image | ✅ | | P3 | Telegram stickers emitted as image/webp attachments |
| Sticker-to-image | ✅ | | P3 | Telegram stickers |
### Owner: _Unassigned_
-5
View File
@@ -14,11 +14,6 @@
<a href="https://www.reddit.com/r/ironclawAI/"><img src="https://img.shields.io/badge/Reddit-r%2FironclawAI-FF4500?style=flat&logo=reddit&logoColor=white" alt="Reddit: r/ironclawAI" /></a>
</p>
<p align="center">
<a href="README.md">English</a> |
<a href="README.zh-CN.md">简体中文</a>
</p>
<p align="center">
<a href="#philosophy">Philosophy</a> •
<a href="#features">Features</a> •
-319
View File
@@ -1,319 +0,0 @@
<p align="center">
<img src="ironclaw.png?v=2" alt="IronClaw" width="200"/>
</p>
<h1 align="center">IronClaw</h1>
<p align="center">
<strong>安全可靠的个人 AI 助手,始终站在你这边</strong>
</p>
<p align="center">
<a href="#license"><img src="https://img.shields.io/badge/license-MIT%20OR%20Apache%202.0-blue.svg" alt="License: MIT OR Apache-2.0" /></a>
<a href="https://t.me/ironclawAI"><img src="https://img.shields.io/badge/Telegram-%40ironclawAI-26A5E4?style=flat&logo=telegram&logoColor=white" alt="Telegram: @ironclawAI" /></a>
<a href="https://www.reddit.com/r/ironclawAI/"><img src="https://img.shields.io/badge/Reddit-r%2FironclawAI-FF4500?style=flat&logo=reddit&logoColor=white" alt="Reddit: r/ironclawAI" /></a>
</p>
<p align="center">
<a href="README.md">English</a> |
<a href="README.zh-CN.md">简体中文</a>
</p>
<p align="center">
<a href="#设计理念">设计理念</a> •
<a href="#功能特性">功能特性</a> •
<a href="#安装">安装</a> •
<a href="#配置">配置</a> •
<a href="#安全机制">安全机制</a> •
<a href="#系统架构">系统架构</a>
</p>
---
## 设计理念
IronClaw 基于一个简单的原则:**你的 AI 助手应该为你服务,而不是与你为敌。**
在 AI 系统对数据处理日益不透明、与企业利益捆绑的今天,IronClaw 选择了一条不同的路:
- **数据归你所有** — 所有信息存储在本地,加密保护,始终在你掌控之下
- **透明至上** — 完全开源,可审计,没有隐藏的遥测或数据收集
- **自主扩展** — 随时构建新工具,无需等待供应商更新
- **纵深防御** — 多层安全机制抵御提示注入和数据泄露
IronClaw 是一个你真正可以信赖的 AI 助手,无论是个人生活还是工作。
## 功能特性
### 安全优先
- **WASM 沙箱** — 不受信任的工具在隔离的 WebAssembly 容器中运行,采用基于能力的权限模型
- **凭据保护** — 密钥永远不会暴露给工具;在宿主边界注入并进行泄露检测
- **提示注入防御** — 模式检测、内容清理和策略执行
- **端点白名单** — HTTP 请求仅限于明确批准的主机和路径
### 随时可用
- **多渠道接入** — REPL、HTTP webhook、WASM 渠道(Telegram、Slack)和 Web 网关
- **Docker 沙箱** — 隔离的容器执行,支持每任务令牌和编排器/工作器模式
- **Web 网关** — 浏览器 UI,支持实时 SSE/WebSocket 流式传输
- **定时任务** — Cron 调度、事件触发器、Webhook 处理器,实现后台自动化
- **心跳系统** — 主动后台执行,用于监控和维护任务
- **并行任务** — 使用隔离上下文同时处理多个请求
- **自修复** — 自动检测并恢复卡住的操作
### 自主扩展
- **动态工具构建** — 描述你的需求,IronClaw 会将其构建为 WASM 工具
- **MCP 协议** — 连接模型上下文协议(Model Context Protocol)服务器以获取额外能力
- **插件架构** — 无需重启即可加载新的 WASM 工具和渠道
### 持久记忆
- **混合搜索** — 全文搜索 + 向量搜索,采用倒数排名融合(Reciprocal Rank Fusion
- **工作空间文件系统** — 灵活的基于路径的存储,用于笔记、日志和上下文
- **身份文件** — 跨会话保持一致的个性和偏好设置
## 安装
### 前置要求
- Rust 1.85+
- PostgreSQL 15+,需安装 [pgvector](https://github.com/pgvector/pgvector) 扩展
- NEAR AI 账户(通过设置向导进行身份验证)
## 下载或编译
访问 [Releases 页面](https://github.com/nearai/ironclaw/releases/) 查看最新版本。
<details>
<summary>通过 Windows 安装程序安装 (Windows)</summary>
下载 [Windows 安装程序](https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-x86_64-pc-windows-msvc.msi) 并运行。
</details>
<details>
<summary>通过 PowerShell 脚本安装 (Windows)</summary>
```sh
irm https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.ps1 | iex
```
</details>
<details>
<summary>通过 Shell 脚本安装 (macOS、Linux、Windows/WSL)</summary>
```sh
curl --proto '=https' --tlsv1.2 -LsSf https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.sh | sh
```
</details>
<details>
<summary>通过 Homebrew 安装 (macOS/Linux)</summary>
```sh
brew install ironclaw
```
</details>
<details>
<summary>从源码编译 (Windows、Linux、macOS 上使用 Cargo)</summary>
确保你已安装 [Rust](https://rustup.rs)。
```bash
# 克隆仓库
git clone https://github.com/nearai/ironclaw.git
cd ironclaw
# 编译
cargo build --release
# 运行测试
cargo test
```
如需进行**完整发布构建**(修改了渠道源码后),先运行 `./scripts/build-all.sh` 重新编译渠道。
</details>
### 数据库设置
```bash
# 创建数据库
createdb ironclaw
# 启用 pgvector 扩展
psql ironclaw -c "CREATE EXTENSION IF NOT EXISTS vector;"
```
## 配置
运行设置向导来配置 IronClaw:
```bash
ironclaw onboard
```
向导将引导你完成数据库连接、NEAR AI 身份验证(通过浏览器 OAuth)和密钥加密(使用系统钥匙串)。设置会保存在数据库中;引导变量(如 `DATABASE_URL``LLM_BACKEND`)写入 `~/.ironclaw/.env`,以便在数据库连接前可用。
### 替代 LLM 提供商
IronClaw 默认使用 NEAR AI,但兼容任何 OpenAI 兼容的端点。
常用选项包括 **OpenRouter**300+ 模型)、**Together AI**、**Fireworks AI**、**Ollama**(本地部署)以及自托管服务器如 **vLLM****LiteLLM**
在向导中选择 *"OpenAI-compatible"*,或直接设置环境变量:
```env
LLM_BACKEND=openai_compatible
LLM_BASE_URL=https://openrouter.ai/api/v1
LLM_API_KEY=sk-or-...
LLM_MODEL=anthropic/claude-sonnet-4
```
详见 [docs/LLM_PROVIDERS.md](docs/LLM_PROVIDERS.md) 获取完整的提供商指南。
## 安全机制
IronClaw 实现了纵深防御策略来保护你的数据并防止滥用。
### WASM 沙箱
所有不受信任的工具都在隔离的 WebAssembly 容器中运行:
- **基于能力的权限** — 明确授权 HTTP、密钥、工具调用等能力
- **端点白名单** — HTTP 请求仅限已批准的主机和路径
- **凭据注入** — 密钥在宿主边界注入,永远不会暴露给 WASM 代码
- **泄露检测** — 扫描请求和响应以防止密钥外泄
- **速率限制** — 每个工具独立的请求限制,防止滥用
- **资源限制** — 内存、CPU 和执行时间约束
```
WASM ──► 白名单 ──► 泄露扫描 ──► 凭据 ──► 执行 ──► 泄露扫描 ──► WASM
验证器 (请求) 注入器 请求 (响应)
```
### 提示注入防御
外部内容需通过多个安全层:
- 基于模式的注入尝试检测
- 内容清理和转义
- 带严重级别的策略规则(阻止/警告/审核/清理)
- 工具输出包装,确保安全的 LLM 上下文注入
### 数据保护
- 所有数据存储在本地 PostgreSQL 数据库中
- 密钥使用 AES-256-GCM 加密
- 无遥测、无分析、无数据共享
- 所有工具执行的完整审计日志
## 系统架构
```
┌────────────────────────────────────────────────────────────────┐
│ 渠道 │
│ ┌──────┐ ┌──────┐ ┌─────────────┐ ┌─────────────┐ │
│ │ REPL │ │ HTTP │ │ WASM 渠道 │ │ Web 网关 │ │
│ └──┬───┘ └──┬───┘ └──────┬──────┘ │ (SSE + WS) │ │
│ │ │ │ └──────┬──────┘ │
│ └─────────┴──────────────┴────────────────┘ │
│ │ │
│ ┌─────────▼─────────┐ │
│ │ 代理循环 │ 意图路由 │
│ └────┬──────────┬───┘ │
│ │ │ │
│ ┌──────────▼────┐ ┌──▼───────────────┐ │
│ │ 调度器 │ │ 定时任务引擎 │ │
│ │ (并行任务) │ │(cron, 事件, wh) │ │
│ └──────┬────────┘ └────────┬─────────┘ │
│ │ │ │
│ ┌─────────────┼────────────────────┘ │
│ │ │ │
│ ┌───▼─────┐ ┌────▼────────────────┐ │
│ │ 本地 │ │ 编排器 │ │
│ │ 工作器 │ │ ┌───────────────┐ │ │
│ │(进程内) │ │ │ Docker 沙箱 │ │ │
│ └───┬─────┘ │ │ 容器 │ │ │
│ │ │ │ ┌───────────┐ │ │ │
│ │ │ │ │工作器/CC │ │ │ │
│ │ │ │ └───────────┘ │ │ │
│ │ │ └───────────────┘ │ │
│ │ └─────────┬───────────┘ │
│ └──────────────────┤ │
│ │ │
│ ┌───────────▼──────────┐ │
│ │ 工具注册表 │ │
│ │ 内置、MCP、WASM │ │
│ └──────────────────────┘ │
└────────────────────────────────────────────────────────────────┘
```
### 核心组件
| 组件 | 用途 |
|------|------|
| **代理循环** | 主消息处理和任务协调 |
| **路由器** | 分类用户意图(命令、查询、任务) |
| **调度器** | 管理带优先级的并行任务执行 |
| **工作器** | 执行包含 LLM 推理和工具调用的任务 |
| **编排器** | 容器生命周期、LLM 代理、每任务认证 |
| **Web 网关** | 浏览器 UI,含聊天、记忆、任务、日志、扩展、定时任务 |
| **定时任务引擎** | 定时(cron)和响应式(事件、webhook)后台任务 |
| **工作空间** | 带混合搜索的持久记忆 |
| **安全层** | 提示注入防御和内容清理 |
## 使用方式
```bash
# 首次设置(配置数据库、认证等)
ironclaw onboard
# 启动交互式 REPL
cargo run
# 启用调试日志
RUST_LOG=ironclaw=debug cargo run
```
## 开发
```bash
# 格式化代码
cargo fmt
# 代码检查
cargo clippy --all --benches --tests --examples --all-features
# 运行测试
createdb ironclaw_test
cargo test
# 运行指定测试
cargo test test_name
```
- **Telegram 渠道**:参见 [docs/TELEGRAM_SETUP.md](docs/TELEGRAM_SETUP.md) 了解设置和私信配对。
- **修改渠道源码**:在 `cargo build` 之前运行 `./channels-src/telegram/build.sh` 以便打包更新后的 WASM。
## OpenClaw 传承
IronClaw 是受 [OpenClaw](https://github.com/openclaw/openclaw) 启发的 Rust 重新实现。参见 [FEATURE_PARITY.md](FEATURE_PARITY.md) 了解完整的功能追踪矩阵。
主要差异:
- **Rust vs TypeScript** — 原生性能、内存安全、单一二进制文件
- **WASM 沙箱 vs Docker** — 轻量级、基于能力的安全机制
- **PostgreSQL vs SQLite** — 生产级持久化存储
- **安全优先设计** — 多层防御、凭据保护
## 许可证
可选择以下任一许可证:
- Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE))
- MIT License ([LICENSE-MIT](LICENSE-MIT))
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "discord-channel"
version = "0.2.0"
version = "0.1.0"
edition = "2021"
description = "Discord channel for IronClaw"
license = "MIT OR Apache-2.0"
@@ -1,6 +1,4 @@
{
"version": "0.2.0",
"wit_version": "0.3.0",
"type": "channel",
"name": "discord",
"description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages",
@@ -8,16 +6,15 @@
"required_secrets": [
{
"name": "discord_bot_token",
"prompt": "Enter your Discord Bot Token. Find it under Bot > Token in your Discord Application settings.",
"prompt": "Enter your Discord Bot Token (from Developer Portal)",
"optional": false
},
{
"name": "discord_public_key",
"prompt": "Enter your Discord Application Public Key (found under General Information in your Discord Application settings).",
"prompt": "Enter your Discord Application Public Key (from Developer Portal > General Information)",
"optional": false
}
],
"setup_url": "https://discord.com/developers/applications"
]
},
"capabilities": {
"http": {
-36
View File
@@ -312,10 +312,6 @@ impl Guest for DiscordChannel {
fn on_status(_update: StatusUpdate) {}
fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> {
Err("broadcast not yet implemented for Discord channel".to_string())
}
fn on_shutdown() {
channel_host::log(
channel_host::LogLevel::Info,
@@ -418,7 +414,6 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
content,
thread_id: None,
metadata_json,
attachments: vec![],
});
true
}
@@ -472,7 +467,6 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
content: format!("[Button clicked] {}", message.content),
thread_id: None,
metadata_json,
attachments: vec![],
});
}
@@ -689,34 +683,4 @@ mod tests {
assert_eq!(parsed.channel_id, "123");
assert_eq!(parsed.interaction_id, "456");
}
#[test]
fn test_parse_slash_command_interaction() {
// Verify that a slash command interaction deserializes correctly.
let json = r#"{
"type": 2,
"id": "int_1",
"application_id": "app_1",
"channel_id": "ch_1",
"member": {
"user": {
"id": "user_1",
"username": "testuser",
"global_name": "Test User"
}
},
"data": {
"id": "cmd_1",
"name": "ask",
"options": [
{"name": "question", "value": "What is rust?"}
]
},
"token": "token_abc"
}"#;
let interaction: DiscordInteraction = serde_json::from_str(json).unwrap();
assert_eq!(interaction.interaction_type, 2);
assert!(interaction.data.is_some());
}
}
+1 -1
View File
@@ -267,7 +267,7 @@ dependencies = [
[[package]]
name = "slack-channel"
version = "0.2.1"
version = "0.1.0"
dependencies = [
"hex",
"hmac",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "slack-channel"
version = "0.2.1"
version = "0.1.0"
edition = "2021"
description = "Slack Events API channel for IronClaw"
license = "MIT OR Apache-2.0"
+3 -9
View File
@@ -1,6 +1,4 @@
{
"version": "0.2.0",
"wit_version": "0.3.0",
"type": "channel",
"name": "slack",
"description": "Slack Events API channel for receiving and responding to Slack messages",
@@ -8,16 +6,15 @@
"required_secrets": [
{
"name": "slack_bot_token",
"prompt": "Enter your Slack Bot User OAuth Token (starts with xoxb-). Find it under OAuth & Permissions in your Slack App settings.",
"prompt": "Enter your Slack Bot OAuth Token (xoxb-...)",
"optional": false
},
{
"name": "slack_signing_secret",
"prompt": "Enter your Slack App Signing Secret (found under Basic Information > App Credentials in your Slack App settings).",
"prompt": "Enter your Slack Signing Secret (from App Credentials)",
"optional": false
}
],
"setup_url": "https://api.slack.com/apps"
]
},
"capabilities": {
"http": {
@@ -46,9 +43,6 @@
"emit_rate_limit": {
"messages_per_minute": 100,
"messages_per_hour": 5000
},
"webhook": {
"hmac_secret_name": "slack_signing_secret"
}
}
},
+3 -279
View File
@@ -29,7 +29,7 @@ use exports::near::agent::channel::{
AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest,
OutgoingHttpResponse, StatusUpdate,
};
use near::agent::channel_host::{self, EmittedMessage, InboundAttachment};
use near::agent::channel_host::{self, EmittedMessage};
/// Slack event wrapper.
#[derive(Debug, Deserialize)]
@@ -78,25 +78,6 @@ struct SlackEvent {
/// Subtype (bot_message, etc.)
subtype: Option<String>,
/// File attachments shared in the message.
#[serde(default)]
files: Option<Vec<SlackFile>>,
}
/// Slack file attachment.
#[derive(Debug, Deserialize)]
struct SlackFile {
/// File ID.
id: String,
/// MIME type.
mimetype: Option<String>,
/// Original filename.
name: Option<String>,
/// File size in bytes.
size: Option<u64>,
/// URL to download the file (requires auth).
url_private: Option<String>,
}
/// Metadata stored with emitted messages for response routing.
@@ -325,140 +306,13 @@ impl Guest for SlackChannel {
fn on_status(_update: StatusUpdate) {}
fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> {
Err("broadcast not yet implemented for Slack channel".to_string())
}
fn on_shutdown() {
channel_host::log(channel_host::LogLevel::Info, "Slack channel shutting down");
}
}
/// Extract attachments from Slack file objects.
fn extract_slack_attachments(files: &Option<Vec<SlackFile>>) -> Vec<InboundAttachment> {
let Some(files) = files else {
return Vec::new();
};
files
.iter()
.map(|f| InboundAttachment {
id: f.id.clone(),
mime_type: f
.mimetype
.clone()
.unwrap_or_else(|| "application/octet-stream".to_string()),
filename: f.name.clone(),
size_bytes: f.size,
source_url: f.url_private.clone(),
storage_key: None,
extracted_text: None,
extras_json: String::new(),
})
.collect()
}
/// Download a file from Slack using the url_private endpoint.
///
/// Slack file downloads require Bearer auth with the bot token, which is
/// injected by the host credential system via `channel_host::http_request`.
fn download_slack_file(url: &str) -> Result<Vec<u8>, String> {
let headers = serde_json::json!({});
let result = channel_host::http_request("GET", url, &headers.to_string(), None, None);
let response = result.map_err(|e| format!("Slack file download failed: {}", e))?;
if response.status != 200 {
let body_str = String::from_utf8_lossy(&response.body);
return Err(format!(
"Slack file download returned {}: {}",
response.status, body_str
));
}
Ok(response.body)
}
/// Download file bytes and store them via the host for processing.
///
/// Downloads all file types (images, documents, etc.) so the host-side
/// middleware can process them (vision pipeline for images, text extraction
/// for documents, transcription for audio, etc.).
/// Maximum file size to download (20 MB). Files larger than this are skipped
/// to avoid excessive memory use and slow downloads in the WASM runtime.
const MAX_DOWNLOAD_SIZE_BYTES: u64 = 20 * 1024 * 1024;
fn download_and_store_slack_files(attachments: &[InboundAttachment]) {
for att in attachments {
let Some(ref url) = att.source_url else {
continue;
};
// Skip files that exceed the size limit
if let Some(size) = att.size_bytes {
if size > MAX_DOWNLOAD_SIZE_BYTES {
channel_host::log(
channel_host::LogLevel::Warn,
&format!(
"Skipping Slack file download: {} bytes exceeds {} MB limit (id={})",
size,
MAX_DOWNLOAD_SIZE_BYTES / (1024 * 1024),
att.id
),
);
continue;
}
}
match download_slack_file(url) {
Ok(bytes) => {
// Post-download size guard: metadata size_bytes is optional,
// so a file with no size info could bypass the pre-download check.
if bytes.len() as u64 > MAX_DOWNLOAD_SIZE_BYTES {
channel_host::log(
channel_host::LogLevel::Warn,
&format!(
"Discarding Slack file after download: {} bytes exceeds {} MB limit (id={})",
bytes.len(),
MAX_DOWNLOAD_SIZE_BYTES / (1024 * 1024),
att.id
),
);
continue;
}
channel_host::log(
channel_host::LogLevel::Info,
&format!(
"Downloaded Slack file: {} bytes, mime={}",
bytes.len(),
att.mime_type
),
);
if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Failed to store Slack file data: {}", e),
);
}
}
Err(e) => {
channel_host::log(
channel_host::LogLevel::Error,
&format!("Failed to download Slack file: {}", e),
);
}
}
}
}
/// Handle a Slack event and emit message if applicable.
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
let attachments = extract_slack_attachments(&event.files);
// Download and store file attachments for host-side processing
download_and_store_slack_files(&attachments);
match event.event_type.as_str() {
// Direct mention of the bot (always in a channel, not a DM)
"app_mention" => {
@@ -472,14 +326,7 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
if !check_sender_permission(&user, &channel, false) {
return;
}
emit_message(
user,
text,
channel,
event.thread_ts.or(Some(ts)),
team_id,
attachments,
);
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
}
}
@@ -501,14 +348,7 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
if !check_sender_permission(&user, &channel, true) {
return;
}
emit_message(
user,
text,
channel,
event.thread_ts.or(Some(ts)),
team_id,
attachments,
);
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
}
}
}
@@ -529,7 +369,6 @@ fn emit_message(
channel: String,
thread_ts: Option<String>,
team_id: Option<String>,
attachments: Vec<InboundAttachment>,
) {
let message_ts = thread_ts.clone().unwrap_or_default();
@@ -557,7 +396,6 @@ fn emit_message(
content: cleaned_text,
thread_id: thread_ts,
metadata_json,
attachments,
});
}
@@ -713,117 +551,3 @@ fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse
// Export the component
export!(SlackChannel);
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_slack_attachments_with_files() {
let files = Some(vec![
SlackFile {
id: "F123".to_string(),
mimetype: Some("image/png".to_string()),
name: Some("screenshot.png".to_string()),
size: Some(50000),
url_private: Some("https://files.slack.com/F123".to_string()),
},
SlackFile {
id: "F456".to_string(),
mimetype: Some("application/pdf".to_string()),
name: Some("doc.pdf".to_string()),
size: Some(120000),
url_private: None,
},
]);
let attachments = extract_slack_attachments(&files);
assert_eq!(attachments.len(), 2);
assert_eq!(attachments[0].id, "F123");
assert_eq!(attachments[0].mime_type, "image/png");
assert_eq!(attachments[0].filename, Some("screenshot.png".to_string()));
assert_eq!(attachments[0].size_bytes, Some(50000));
assert_eq!(
attachments[0].source_url,
Some("https://files.slack.com/F123".to_string())
);
assert_eq!(attachments[1].id, "F456");
assert_eq!(attachments[1].mime_type, "application/pdf");
assert!(attachments[1].source_url.is_none());
}
#[test]
fn test_extract_slack_attachments_none() {
let attachments = extract_slack_attachments(&None);
assert!(attachments.is_empty());
}
#[test]
fn test_extract_slack_attachments_empty() {
let attachments = extract_slack_attachments(&Some(vec![]));
assert!(attachments.is_empty());
}
#[test]
fn test_extract_slack_attachments_missing_mime() {
let files = Some(vec![SlackFile {
id: "F789".to_string(),
mimetype: None,
name: Some("unknown".to_string()),
size: None,
url_private: None,
}]);
let attachments = extract_slack_attachments(&files);
assert_eq!(attachments.len(), 1);
assert_eq!(attachments[0].mime_type, "application/octet-stream");
}
#[test]
fn test_parse_slack_event_with_files() {
let json = r#"{
"type": "message",
"user": "U123",
"channel": "D456",
"text": "Check this file",
"ts": "1234567890.000001",
"files": [
{
"id": "F001",
"mimetype": "image/jpeg",
"name": "photo.jpg",
"size": 30000,
"url_private": "https://files.slack.com/F001"
}
]
}"#;
let event: SlackEvent = serde_json::from_str(json).unwrap();
assert!(event.files.is_some());
let files = event.files.unwrap();
assert_eq!(files.len(), 1);
assert_eq!(files[0].id, "F001");
}
#[test]
fn test_parse_slack_event_without_files() {
let json = r#"{
"type": "message",
"user": "U123",
"channel": "D456",
"text": "Just text",
"ts": "1234567890.000001"
}"#;
let event: SlackEvent = serde_json::from_str(json).unwrap();
assert!(event.files.is_none());
}
#[test]
fn test_max_download_size_constant() {
// Verify the constant is 20 MB
assert_eq!(MAX_DOWNLOAD_SIZE_BYTES, 20 * 1024 * 1024);
}
}
+1 -1
View File
@@ -212,7 +212,7 @@ dependencies = [
[[package]]
name = "telegram-channel"
version = "0.2.1"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "telegram-channel"
version = "0.2.1"
version = "0.1.0"
edition = "2021"
description = "Telegram Bot API channel for IronClaw"
license = "MIT OR Apache-2.0"
File diff suppressed because it is too large Load Diff
@@ -1,17 +1,7 @@
{
"version": "0.2.2",
"wit_version": "0.3.0",
"type": "channel",
"name": "telegram",
"description": "Telegram Bot API channel for receiving and responding to Telegram messages",
"auth": {
"secret_name": "telegram_bot_token",
"display_name": "Telegram",
"instructions": "Get your bot token from @BotFather on Telegram (https://t.me/BotFather). Send /newbot or /token to get it.",
"setup_url": "https://t.me/BotFather",
"token_hint": "Looks like 123456789:AABBccDDeeFFgg...",
"env_var": "TELEGRAM_BOT_TOKEN"
},
"setup": {
"required_secrets": [
{
@@ -19,14 +9,12 @@
"prompt": "Enter your Telegram Bot API token (from @BotFather)",
"optional": false
}
],
"setup_url": "https://t.me/BotFather"
]
},
"capabilities": {
"http": {
"allowlist": [
{ "host": "api.telegram.org", "path_prefix": "/bot" },
{ "host": "api.telegram.org", "path_prefix": "/file/bot" }
{ "host": "api.telegram.org", "path_prefix": "/bot" }
],
"credentials": {
"telegram_bot": {
@@ -35,7 +23,6 @@
"host_patterns": ["api.telegram.org"]
}
},
"max_response_bytes": 52428800,
"rate_limit": {
"requests_per_minute": 30,
"requests_per_hour": 1000
@@ -52,10 +39,6 @@
"emit_rate_limit": {
"messages_per_minute": 100,
"messages_per_hour": 5000
},
"webhook": {
"secret_header": "X-Telegram-Bot-Api-Secret-Token",
"secret_name": "telegram_webhook_secret"
}
}
},
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "whatsapp-channel"
version = "0.2.0"
version = "0.1.0"
edition = "2021"
description = "WhatsApp Cloud API channel for IronClaw"
+12 -263
View File
@@ -32,7 +32,7 @@ use exports::near::agent::channel::{
AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest,
OutgoingHttpResponse, StatusUpdate,
};
use near::agent::channel_host::{self, EmittedMessage, InboundAttachment};
use near::agent::channel_host::{self, EmittedMessage};
// ============================================================================
// WhatsApp Cloud API Types
@@ -137,46 +137,10 @@ struct WhatsAppMessage {
/// Text content (if type is "text")
text: Option<TextContent>,
/// Image content
image: Option<WhatsAppMedia>,
/// Audio content
audio: Option<WhatsAppMedia>,
/// Video content
video: Option<WhatsAppMedia>,
/// Document content
document: Option<WhatsAppDocument>,
/// Context for replies
context: Option<MessageContext>,
}
/// WhatsApp media attachment (image, audio, video).
#[derive(Debug, Deserialize)]
struct WhatsAppMedia {
/// Media ID (use to download via Graph API)
id: String,
/// MIME type
mime_type: Option<String>,
/// Caption text
caption: Option<String>,
}
/// WhatsApp document attachment.
#[derive(Debug, Deserialize)]
struct WhatsAppDocument {
/// Media ID
id: String,
/// MIME type
mime_type: Option<String>,
/// Filename
filename: Option<String>,
/// Caption text
caption: Option<String>,
}
/// Text message content.
#[derive(Debug, Deserialize)]
struct TextContent {
@@ -512,10 +476,6 @@ impl Guest for WhatsAppChannel {
fn on_status(_update: StatusUpdate) {}
fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> {
Err("broadcast not yet implemented for WhatsApp channel".to_string())
}
fn on_shutdown() {
channel_host::log(
channel_host::LogLevel::Info,
@@ -658,102 +618,26 @@ fn handle_incoming_message(req: &IncomingHttpRequest) -> OutgoingHttpResponse {
json_response(200, serde_json::json!({"status": "ok"}))
}
/// Extract attachments from a WhatsApp message.
fn extract_whatsapp_attachments(message: &WhatsAppMessage) -> Vec<InboundAttachment> {
let mut attachments = Vec::new();
if let Some(ref img) = message.image {
attachments.push(InboundAttachment {
id: img.id.clone(),
mime_type: img
.mime_type
.clone()
.unwrap_or_else(|| "image/jpeg".to_string()),
filename: None,
size_bytes: None,
source_url: None, // WhatsApp requires Graph API call with media ID to get URL
storage_key: None,
extracted_text: img.caption.clone(),
extras_json: String::new(),
});
}
if let Some(ref audio) = message.audio {
attachments.push(InboundAttachment {
id: audio.id.clone(),
mime_type: audio
.mime_type
.clone()
.unwrap_or_else(|| "audio/ogg".to_string()),
filename: None,
size_bytes: None,
source_url: None,
storage_key: None,
extracted_text: audio.caption.clone(),
extras_json: String::new(),
});
}
if let Some(ref video) = message.video {
attachments.push(InboundAttachment {
id: video.id.clone(),
mime_type: video
.mime_type
.clone()
.unwrap_or_else(|| "video/mp4".to_string()),
filename: None,
size_bytes: None,
source_url: None,
storage_key: None,
extracted_text: video.caption.clone(),
extras_json: String::new(),
});
}
if let Some(ref doc) = message.document {
attachments.push(InboundAttachment {
id: doc.id.clone(),
mime_type: doc
.mime_type
.clone()
.unwrap_or_else(|| "application/octet-stream".to_string()),
filename: doc.filename.clone(),
size_bytes: None,
source_url: None,
storage_key: None,
extracted_text: doc.caption.clone(),
extras_json: String::new(),
});
}
attachments
}
/// Process a single WhatsApp message.
fn handle_message(
message: &WhatsAppMessage,
phone_number_id: &str,
contact_names: &std::collections::HashMap<String, String>,
) {
let attachments = extract_whatsapp_attachments(message);
// Only handle text messages for now
// TODO: Add support for image, audio, video, document, etc.
if message.message_type != "text" {
channel_host::log(
channel_host::LogLevel::Debug,
&format!("Skipping non-text message type: {}", message.message_type),
);
return;
}
// Extract text content (from text body or media captions)
// Extract text content
let text = match &message.text {
Some(t) if !t.body.is_empty() => t.body.clone(),
_ => {
// Try to use caption from media messages as content
let caption = message
.image
.as_ref()
.and_then(|m| m.caption.clone())
.or_else(|| message.video.as_ref().and_then(|m| m.caption.clone()))
.or_else(|| message.document.as_ref().and_then(|m| m.caption.clone()));
match caption {
Some(c) if !c.is_empty() => c,
_ if !attachments.is_empty() => String::new(),
_ => return,
}
}
_ => return,
};
// Look up sender's name from contacts
@@ -786,7 +670,6 @@ fn handle_message(
content: text,
thread_id: None, // WhatsApp doesn't have threads like Slack/Discord
metadata_json,
attachments,
});
channel_host::log(
@@ -1064,138 +947,4 @@ mod tests {
assert_eq!(parsed.phone_number_id, "123456");
assert_eq!(parsed.sender_phone, "15551234567");
}
// === Attachment extraction fixture tests ===
#[test]
fn test_extract_whatsapp_image_attachment() {
let msg = WhatsAppMessage {
id: "msg1".to_string(),
from: "15551234567".to_string(),
timestamp: "1234567890".to_string(),
message_type: "image".to_string(),
text: None,
image: Some(WhatsAppMedia {
id: "media_img_1".to_string(),
mime_type: Some("image/jpeg".to_string()),
caption: Some("Look at this".to_string()),
}),
audio: None,
video: None,
document: None,
context: None,
};
let attachments = extract_whatsapp_attachments(&msg);
assert_eq!(attachments.len(), 1);
assert_eq!(attachments[0].id, "media_img_1");
assert_eq!(attachments[0].mime_type, "image/jpeg");
assert_eq!(
attachments[0].extracted_text,
Some("Look at this".to_string())
);
}
#[test]
fn test_extract_whatsapp_document_attachment() {
let msg = WhatsAppMessage {
id: "msg2".to_string(),
from: "15551234567".to_string(),
timestamp: "1234567890".to_string(),
message_type: "document".to_string(),
text: None,
image: None,
audio: None,
video: None,
document: Some(WhatsAppDocument {
id: "media_doc_1".to_string(),
mime_type: Some("application/pdf".to_string()),
filename: Some("report.pdf".to_string()),
caption: None,
}),
context: None,
};
let attachments = extract_whatsapp_attachments(&msg);
assert_eq!(attachments.len(), 1);
assert_eq!(attachments[0].id, "media_doc_1");
assert_eq!(attachments[0].mime_type, "application/pdf");
assert_eq!(
attachments[0].filename,
Some("report.pdf".to_string())
);
}
#[test]
fn test_extract_whatsapp_audio_video_attachments() {
let msg = WhatsAppMessage {
id: "msg3".to_string(),
from: "15551234567".to_string(),
timestamp: "1234567890".to_string(),
message_type: "audio".to_string(),
text: None,
image: None,
audio: Some(WhatsAppMedia {
id: "media_audio_1".to_string(),
mime_type: Some("audio/ogg".to_string()),
caption: None,
}),
video: Some(WhatsAppMedia {
id: "media_video_1".to_string(),
mime_type: Some("video/mp4".to_string()),
caption: None,
}),
document: None,
context: None,
};
let attachments = extract_whatsapp_attachments(&msg);
assert_eq!(attachments.len(), 2);
assert_eq!(attachments[0].id, "media_audio_1");
assert_eq!(attachments[1].id, "media_video_1");
}
#[test]
fn test_extract_whatsapp_text_only_no_attachments() {
let msg = WhatsAppMessage {
id: "msg4".to_string(),
from: "15551234567".to_string(),
timestamp: "1234567890".to_string(),
message_type: "text".to_string(),
text: Some(TextContent {
body: "Hello".to_string(),
}),
image: None,
audio: None,
video: None,
document: None,
context: None,
};
let attachments = extract_whatsapp_attachments(&msg);
assert!(attachments.is_empty());
}
#[test]
fn test_parse_whatsapp_image_message() {
let json = r#"{
"id": "wamid.123",
"from": "15551234567",
"timestamp": "1234567890",
"type": "image",
"image": {
"id": "media_img_abc",
"mime_type": "image/jpeg",
"caption": "Check this"
}
}"#;
let msg: WhatsAppMessage = serde_json::from_str(json).unwrap();
assert_eq!(msg.message_type, "image");
assert!(msg.image.is_some());
let attachments = extract_whatsapp_attachments(&msg);
assert_eq!(attachments.len(), 1);
assert_eq!(attachments[0].id, "media_img_abc");
}
}
@@ -1,6 +1,4 @@
{
"version": "0.2.0",
"wit_version": "0.3.0",
"type": "channel",
"name": "whatsapp",
"description": "WhatsApp Cloud API channel for receiving and responding to WhatsApp messages",
@@ -8,7 +6,7 @@
"required_secrets": [
{
"name": "whatsapp_access_token",
"prompt": "Enter your WhatsApp Cloud API permanent access token (from the Meta Developer Portal under your app's WhatsApp > API Setup).",
"prompt": "Enter your WhatsApp Cloud API access token (from Meta Developer Portal)",
"validation": "^[A-Za-z0-9_-]+$"
},
{
@@ -18,8 +16,7 @@
"auto_generate": { "length": 32 }
}
],
"validation_endpoint": "https://graph.facebook.com/v18.0/me?access_token={whatsapp_access_token}",
"setup_url": "https://developers.facebook.com/apps"
"validation_endpoint": "https://graph.facebook.com/v18.0/me?access_token={whatsapp_access_token}"
},
"capabilities": {
"http": {
-10
View File
@@ -1,10 +0,0 @@
coverage:
status:
project:
default:
target: auto
threshold: 1%
patch:
default:
target: 80%
threshold: 5%
-9
View File
@@ -24,15 +24,6 @@ GATEWAY_HOST=0.0.0.0
GATEWAY_PORT=3000
GATEWAY_AUTH_TOKEN=CHANGE_ME
# Restart Feature (Docker containers only)
# IMPORTANT: Set this in the container entrypoint or docker-compose to enable restart.
# The Docker entrypoint loop monitors exit codes:
# - Exit code 0 = clean restart: reset failure counter, wait IRONCLAW_RESTART_DELAY, restart
# - Exit code ≠ 0 = failure: increment counter, exit after IRONCLAW_MAX_FAILURES
IRONCLAW_IN_DOCKER=false
IRONCLAW_RESTART_DELAY=5 # seconds to wait before restarting (range: 1-30)
IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits
# Disabled for initial deploy
SANDBOX_ENABLED=false
HEARTBEAT_ENABLED=false
+1 -1
View File
@@ -3,7 +3,7 @@ services:
postgres:
image: pgvector/pgvector:pg16
ports:
- "127.0.0.1:5432:5432"
- "5432:5432"
environment:
POSTGRES_DB: ironclaw
POSTGRES_USER: ironclaw
-55
View File
@@ -11,13 +11,7 @@ configurations.
| NEAR AI | `nearai` | OAuth (browser) | Default; multi-model |
| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | Claude models |
| OpenAI | `openai` | `OPENAI_API_KEY` | GPT models |
| Google Gemini | `gemini` | `GEMINI_API_KEY` | Gemini models |
| io.net | `ionet` | `IONET_API_KEY` | Intelligence API |
| Mistral | `mistral` | `MISTRAL_API_KEY` | Mistral models |
| Yandex AI Studio | `yandex` | `YANDEX_API_KEY` | YandexGPT models |
| Cloudflare Workers AI | `cloudflare` | `CLOUDFLARE_API_KEY` | Access to Workers AI |
| Ollama | `ollama` | No | Local inference |
| AWS Bedrock | `bedrock` | AWS credentials | Native Converse API |
| OpenRouter | `openai_compatible` | `LLM_API_KEY` | 300+ models |
| Together AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
| Fireworks AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
@@ -74,55 +68,6 @@ Pull a model first: `ollama pull llama3.2`
---
## AWS Bedrock (requires `--features bedrock`)
Uses the native AWS Converse API via `aws-sdk-bedrockruntime`. Supports standard AWS
authentication methods: IAM credentials, SSO profiles, and instance roles.
> **Build prerequisite:** The `aws-lc-sys` crate (transitive dependency via AWS SDK)
> requires **CMake** to compile. Install it before building with `--features bedrock`:
> - macOS: `brew install cmake`
> - Ubuntu/Debian: `sudo apt install cmake`
> - Fedora: `sudo dnf install cmake`
### With AWS credentials (IAM, SSO, instance roles)
```env
LLM_BACKEND=bedrock
BEDROCK_MODEL=anthropic.claude-opus-4-6-v1
BEDROCK_REGION=us-east-1
BEDROCK_CROSS_REGION=us
# AWS_PROFILE=my-sso-profile # optional, for named profiles
```
The AWS SDK credential chain automatically resolves credentials from environment
variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`), shared credentials file
(`~/.aws/credentials`), SSO profiles, and EC2/ECS instance roles.
### Cross-region inference
Set `BEDROCK_CROSS_REGION` to route requests across AWS regions for capacity:
| Prefix | Routing |
|---|---|
| `us` | US regions (us-east-1, us-east-2, us-west-2) |
| `eu` | European regions |
| `apac` | Asia-Pacific regions |
| `global` | All commercial AWS regions |
| _(unset)_ | Single-region only |
### Popular Bedrock model IDs
| Model | ID |
|---|---|
| Claude Opus 4.6 | `anthropic.claude-opus-4-6-v1` |
| Claude Sonnet 4.5 | `anthropic.claude-sonnet-4-5-20250929-v1:0` |
| Claude Haiku 4.5 | `anthropic.claude-haiku-4-5-20251001-v1:0` |
| Amazon Nova Pro | `amazon.nova-pro-v1:0` |
| Llama 4 Maverick | `meta.llama4-maverick-17b-instruct-v1:0` |
---
## OpenAI-Compatible Endpoints
All providers below use `LLM_BACKEND=openai_compatible`. Set `LLM_BASE_URL` to the
-195
View File
@@ -1,195 +0,0 @@
# Smart Model Routing for IronClaw
**Status:** Implemented
**Author:** Microwave
**Date:** 2026-02-19
## What
Automatic model selection based on request complexity. The router analyzes each user message and selects an appropriate model tier (flash/standard/pro/frontier), then maps that tier to a configured model.
## Why
1. **Cost optimization** — Simple requests ("hi", "what time is it") don't need expensive models
2. **User experience** — Simple requests return faster with lightweight models
3. **NEAR AI native** — Default backend uses NEAR AI inference where costs vary by model
4. **Zero-config value** — Users benefit immediately without configuration
5. **Not just power users** — Everyone gets smart defaults, power users can override
## How
### Architecture
```
User Message
┌──────────────────┐
│ Pattern Overrides │ ← Fast-path for obvious cases (greetings, security audits)
└────────┬─────────┘
│ no match
┌──────────────────┐
│ Complexity Scorer │ ← 13-dimension analysis
└────────┬─────────┘
│ score 0-100
┌──────────────────┐
│ Tier Mapping │ ← 0-15: flash, 16-40: standard, 41-65: pro, 66+: frontier
└────────┬─────────┘
│ tier
┌──────────────────┐
│ Model Selection │ ← Currently: cheap provider (Flash/Standard/Pro) vs primary (Frontier)
└────────┬─────────┘ Target: per-tier model mapping via config
LLM Provider
```
### Complexity Scorer (13 Dimensions)
Each dimension produces a 0-100 score. Weighted sum determines total.
| Dimension | Weight | Signals |
|-----------|--------|---------|
| Reasoning Words | 14% | "why", "explain", "compare", "trade-offs" |
| Token Estimate | 12% | Prompt length |
| Code Indicators | 10% | Backticks, syntax, "implement", "PR" |
| Multi-Step | 10% | "first", "then", "after", "steps" |
| Domain Specific | 10% | Technical terms (configurable) |
| Creativity | 7% | "write", "summarize", "tweet", "blog" |
| Question Complexity | 7% | Multiple questions, open-ended starters |
| Precision | 6% | Numbers, "exactly", "calculate" |
| Ambiguity | 5% | Vague references |
| Context Dependency | 5% | "previous", "you said" |
| Sentence Complexity | 5% | Commas, conjunctions, clause depth |
| Tool Likelihood | 5% | "read", "deploy", "install" |
| Safety Sensitivity | 4% | "password", "auth", "vulnerability" |
**Multi-dimensional boost:** +30% when 3+ dimensions score above threshold.
### Tier Boundaries
| Score | Tier | Typical Use Case |
|-------|------|------------------|
| 0-15 | flash | Greetings, acknowledgments, quick lookups |
| 16-40 | standard | Writing, comparisons, defined tasks |
| 41-65 | pro | Multi-step analysis, code review |
| 66+ | frontier | Critical decisions, security audits |
### Pattern Overrides
Fast-path rules that bypass scoring for obvious cases:
```yaml
# Force flash tier
- "^(hi|hello|hey|thanks|ok|sure|yes|no)$"
- "^what.*(time|date|day)"
# Force frontier tier
- "security.*(audit|review|scan)"
- "vulnerabilit(y|ies).*(review|scan|check|audit)"
# Force pro tier
- "deploy.*(mainnet|production)"
```
### Configuration
> **Note:** The current implementation supports smart routing via
> `NEARAI_CHEAP_MODEL` and `SMART_ROUTING_CASCADE` env vars, plus
> `domain_keywords` on `SmartRoutingConfig`. The full `llm.routing` YAML
> schema below is the target design — not all knobs are wired yet.
**Default (zero-config):**
```yaml
llm:
routing:
enabled: true # default
```
**Power user overrides (target schema):**
```yaml
llm:
routing:
enabled: true
tiers:
flash: "claude-3-5-haiku-latest"
standard: "claude-sonnet-4-5-latest"
pro: "claude-sonnet-4-5-latest"
frontier: "claude-opus-4-5-latest"
thinking:
pro: "low"
frontier: "medium"
overrides:
- pattern: "my-custom-pattern"
tier: "pro"
domain_keywords: # Custom keywords for your domain
- "mycompany"
- "myproduct"
- "internal-tool"
```
If `domain_keywords` is not set, uses `DEFAULT_DOMAIN_KEYWORDS` which covers common web3/infra terms.
**Disable routing (pin model):**
```yaml
llm:
routing:
enabled: false
model: "claude-opus-4-5"
```
**Bring your own keys:**
```yaml
llm:
backend: anthropic
api_key: "sk-..."
routing:
enabled: true # still works with external providers
```
### Integration Points
1. **RoutingProvider** — New wrapper implementing `LlmProvider` trait (like `FailoverProvider`)
2. **Scorer** — Pure function, no I/O, fast (~1ms)
3. **Config schema** — Extend `LlmConfig` with `routing` section
4. **Telemetry** — Log routing decisions for observability
### Model Agnosticism
**Critical:** No hardcoded model names in the router logic itself.
- Tier→model mappings come from config
- Default mappings use `-latest` patterns where supported
- NEAR AI backend handles actual model resolution
- Router only knows about tiers
### Layers of Control
| Layer | User Type | Config |
|-------|-----------|--------|
| 1. Zero-config | Everyone | `routing.enabled: true` (default) |
| 2. Tier tuning | Power users | Custom `routing.tiers` mapping |
| 3. Pattern overrides | Power users | Custom `routing.overrides` |
| 4. Model pinning | Power users | `routing.enabled: false` + `model: X` |
| 5. Own API keys | Power users | `backend: anthropic` + `api_key` |
## Implementation Plan
1. [x] Port scorer to Rust (`src/llm/smart_routing.rs`)
2. [x] Implement router wrapper (`src/llm/smart_routing.rs`)
3. [x] Extend config schema (`src/config.rs`)
4. [x] Wire into provider creation (`src/llm/mod.rs`)
5. [x] Add telemetry/logging
6. [x] Tests with real conversation samples
7. [x] Codex + Gemini security review
8. [x] Documentation updated (this spec)
## Expected Outcomes
- **50-70% cost reduction** for typical usage patterns
- **Faster responses** for simple requests
- **Zero config required** for default benefits
- **Full control** for power users who want it
-19
View File
@@ -1,19 +0,0 @@
-- Add wit_version column to wasm_tools for WIT interface version tracking
ALTER TABLE wasm_tools ADD COLUMN IF NOT EXISTS wit_version TEXT NOT NULL DEFAULT '0.1.0';
-- Create wasm_channels table for DB-stored channel extensions
CREATE TABLE IF NOT EXISTS wasm_channels (
id UUID PRIMARY KEY,
user_id TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL DEFAULT '0.1.0',
wit_version TEXT NOT NULL DEFAULT '0.1.0',
description TEXT NOT NULL DEFAULT '',
wasm_binary BYTEA NOT NULL,
binary_hash BYTEA NOT NULL,
capabilities_json TEXT NOT NULL DEFAULT '{}',
status TEXT NOT NULL DEFAULT 'active',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT unique_wasm_channel UNIQUE (user_id, name)
);
@@ -1,13 +0,0 @@
-- Partial unique indexes to prevent duplicate singleton conversations.
-- These guard against TOCTOU races in get_or_create_routine_conversation
-- and get_or_create_heartbeat_conversation.
-- One routine conversation per user per routine_id.
CREATE UNIQUE INDEX IF NOT EXISTS uq_conv_routine
ON conversations (user_id, (metadata->>'routine_id'))
WHERE metadata->>'routine_id' IS NOT NULL;
-- One heartbeat conversation per user.
CREATE UNIQUE INDEX IF NOT EXISTS uq_conv_heartbeat
ON conversations (user_id)
WHERE metadata->>'thread_type' = 'heartbeat';
-383
View File
@@ -1,383 +0,0 @@
[
{
"id": "openai",
"aliases": [
"open_ai"
],
"protocol": "open_ai_completions",
"api_key_env": "OPENAI_API_KEY",
"api_key_required": true,
"base_url_env": "OPENAI_BASE_URL",
"model_env": "OPENAI_MODEL",
"default_model": "gpt-4o",
"description": "OpenAI GPT models (direct API)",
"setup": {
"kind": "api_key",
"secret_name": "llm_openai_api_key",
"key_url": "https://platform.openai.com/api-keys",
"display_name": "OpenAI",
"can_list_models": true
}
},
{
"id": "anthropic",
"aliases": [
"claude"
],
"protocol": "anthropic",
"api_key_env": "ANTHROPIC_API_KEY",
"api_key_required": true,
"base_url_env": "ANTHROPIC_BASE_URL",
"model_env": "ANTHROPIC_MODEL",
"default_model": "claude-sonnet-4-20250514",
"description": "Anthropic Claude models (direct API)",
"setup": {
"kind": "api_key",
"secret_name": "llm_anthropic_api_key",
"key_url": "https://console.anthropic.com/settings/keys",
"display_name": "Anthropic",
"can_list_models": true
}
},
{
"id": "ollama",
"aliases": [],
"protocol": "ollama",
"default_base_url": "http://localhost:11434",
"base_url_env": "OLLAMA_BASE_URL",
"model_env": "OLLAMA_MODEL",
"default_model": "llama3",
"description": "Local Ollama instance (no API key needed)",
"setup": {
"kind": "ollama",
"display_name": "Ollama",
"can_list_models": true
}
},
{
"id": "openai_compatible",
"aliases": [
"openai-compatible",
"compatible"
],
"protocol": "open_ai_completions",
"base_url_env": "LLM_BASE_URL",
"base_url_required": true,
"api_key_env": "LLM_API_KEY",
"api_key_required": false,
"model_env": "LLM_MODEL",
"default_model": "default",
"extra_headers_env": "LLM_EXTRA_HEADERS",
"description": "Custom OpenAI-compatible endpoint (vLLM, LiteLLM, etc.)",
"setup": {
"kind": "open_ai_compatible",
"secret_name": "llm_compatible_api_key",
"display_name": "OpenAI-compatible",
"can_list_models": false
}
},
{
"id": "tinfoil",
"aliases": [],
"protocol": "open_ai_completions",
"default_base_url": "https://inference.tinfoil.sh/v1",
"api_key_env": "TINFOIL_API_KEY",
"api_key_required": true,
"model_env": "TINFOIL_MODEL",
"default_model": "kimi-k2-5",
"description": "Tinfoil private inference (hardware-attested TEE)",
"setup": {
"kind": "api_key",
"secret_name": "llm_tinfoil_api_key",
"key_url": "https://tinfoil.sh",
"display_name": "Tinfoil",
"can_list_models": false
}
},
{
"id": "openrouter",
"aliases": [
"open_router"
],
"protocol": "open_ai_completions",
"default_base_url": "https://openrouter.ai/api/v1",
"api_key_env": "OPENROUTER_API_KEY",
"api_key_required": true,
"model_env": "OPENROUTER_MODEL",
"default_model": "openai/gpt-4o",
"description": "OpenRouter multi-provider gateway (200+ models)",
"setup": {
"kind": "api_key",
"secret_name": "llm_openrouter_api_key",
"key_url": "https://openrouter.ai/settings/keys",
"display_name": "OpenRouter",
"can_list_models": false
}
},
{
"id": "groq",
"aliases": [],
"protocol": "open_ai_completions",
"default_base_url": "https://api.groq.com/openai/v1",
"api_key_env": "GROQ_API_KEY",
"api_key_required": true,
"model_env": "GROQ_MODEL",
"default_model": "llama-3.3-70b-versatile",
"description": "Groq LPU inference (ultra-fast)",
"setup": {
"kind": "api_key",
"secret_name": "llm_groq_api_key",
"key_url": "https://console.groq.com/keys",
"display_name": "Groq",
"can_list_models": true,
"models_filter": "chat"
}
},
{
"id": "nvidia",
"aliases": [
"nvidia_nim",
"nim"
],
"protocol": "open_ai_completions",
"default_base_url": "https://integrate.api.nvidia.com/v1",
"api_key_env": "NVIDIA_API_KEY",
"api_key_required": true,
"model_env": "NVIDIA_MODEL",
"default_model": "meta/llama-3.3-70b-instruct",
"description": "NVIDIA NIM API (high-performance inference)",
"setup": {
"kind": "api_key",
"secret_name": "llm_nvidia_api_key",
"key_url": "https://build.nvidia.com",
"display_name": "NVIDIA NIM",
"can_list_models": true
}
},
{
"id": "venice",
"aliases": [
"venice_ai",
"veniceai"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.venice.ai/api/v1",
"api_key_env": "VENICE_API_KEY",
"api_key_required": true,
"model_env": "VENICE_MODEL",
"default_model": "llama-3.3-70b",
"description": "Venice.ai privacy-focused inference",
"setup": {
"kind": "api_key",
"secret_name": "llm_venice_api_key",
"key_url": "https://venice.ai/settings/api",
"display_name": "Venice.ai",
"can_list_models": false
}
},
{
"id": "together",
"aliases": [
"together_ai",
"togetherai"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.together.xyz/v1",
"api_key_env": "TOGETHER_API_KEY",
"api_key_required": true,
"model_env": "TOGETHER_MODEL",
"default_model": "meta-llama/Llama-3-70b-chat-hf",
"description": "Together AI inference",
"setup": {
"kind": "api_key",
"secret_name": "llm_together_api_key",
"key_url": "https://api.together.ai/settings/api-keys",
"display_name": "Together AI",
"can_list_models": false
}
},
{
"id": "fireworks",
"aliases": [
"fireworks_ai"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.fireworks.ai/inference/v1",
"api_key_env": "FIREWORKS_API_KEY",
"api_key_required": true,
"model_env": "FIREWORKS_MODEL",
"default_model": "accounts/fireworks/models/llama-v3p1-70b-instruct",
"description": "Fireworks AI inference",
"setup": {
"kind": "api_key",
"secret_name": "llm_fireworks_api_key",
"key_url": "https://fireworks.ai/api-keys",
"display_name": "Fireworks AI",
"can_list_models": false
}
},
{
"id": "deepseek",
"aliases": [
"deep_seek"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.deepseek.com/v1",
"api_key_env": "DEEPSEEK_API_KEY",
"api_key_required": true,
"model_env": "DEEPSEEK_MODEL",
"default_model": "deepseek-chat",
"description": "DeepSeek inference API",
"setup": {
"kind": "api_key",
"secret_name": "llm_deepseek_api_key",
"key_url": "https://platform.deepseek.com/api_keys",
"display_name": "DeepSeek",
"can_list_models": false
}
},
{
"id": "cerebras",
"aliases": [],
"protocol": "open_ai_completions",
"default_base_url": "https://api.cerebras.ai/v1",
"api_key_env": "CEREBRAS_API_KEY",
"api_key_required": true,
"model_env": "CEREBRAS_MODEL",
"default_model": "llama-3.3-70b",
"description": "Cerebras wafer-scale inference",
"setup": {
"kind": "api_key",
"secret_name": "llm_cerebras_api_key",
"key_url": "https://cloud.cerebras.ai",
"display_name": "Cerebras",
"can_list_models": false
}
},
{
"id": "sambanova",
"aliases": [
"samba_nova"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.sambanova.ai/v1",
"api_key_env": "SAMBANOVA_API_KEY",
"api_key_required": true,
"model_env": "SAMBANOVA_MODEL",
"default_model": "Meta-Llama-3.1-70B-Instruct",
"description": "SambaNova Cloud inference",
"setup": {
"kind": "api_key",
"secret_name": "llm_sambanova_api_key",
"key_url": "https://cloud.sambanova.ai/apis",
"display_name": "SambaNova",
"can_list_models": false
}
},
{
"id": "gemini",
"aliases": [
"google_gemini",
"google"
],
"protocol": "open_ai_completions",
"default_base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
"api_key_env": "GEMINI_API_KEY",
"api_key_required": true,
"model_env": "GEMINI_MODEL",
"default_model": "gemini-2.5-flash",
"description": "Google Gemini (via OpenAI-compatible endpoint)",
"setup": {
"kind": "api_key",
"secret_name": "llm_gemini_api_key",
"key_url": "https://aistudio.google.com/app/apikey",
"display_name": "Google Gemini",
"can_list_models": true
}
},
{
"id": "ionet",
"aliases": [
"io_net",
"io.net"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.intelligence.io.solutions/api/v1",
"api_key_env": "IONET_API_KEY",
"api_key_required": true,
"model_env": "IONET_MODEL",
"default_model": "deepseek-coder-v2-instruct",
"description": "io.net Intelligence API",
"setup": {
"kind": "api_key",
"secret_name": "llm_ionet_api_key",
"key_url": "https://cloud.io.net/intelligence",
"display_name": "io.net",
"can_list_models": true
}
},
{
"id": "mistral",
"aliases": [
"mistral_ai",
"mistralai"
],
"protocol": "open_ai_completions",
"default_base_url": "https://api.mistral.ai/v1",
"api_key_env": "MISTRAL_API_KEY",
"api_key_required": true,
"model_env": "MISTRAL_MODEL",
"default_model": "mistral-large-latest",
"description": "Mistral AI API",
"setup": {
"kind": "api_key",
"secret_name": "llm_mistral_api_key",
"key_url": "https://console.mistral.ai/api-keys",
"display_name": "Mistral",
"can_list_models": true
}
},
{
"id": "yandex",
"aliases": [
"yandex_ai_studio",
"yandexgpt",
"yandex_gpt"
],
"protocol": "open_ai_completions",
"default_base_url": "https://ai.api.cloud.yandex.net/v1",
"api_key_env": "YANDEX_API_KEY",
"api_key_required": true,
"model_env": "YANDEX_MODEL",
"extra_headers_env": "YANDEX_EXTRA_HEADERS",
"default_model": "yandexgpt-lite",
"description": "Yandex AI Studio (YandexGPT)",
"setup": {
"kind": "api_key",
"secret_name": "llm_yandex_api_key",
"key_url": "https://aistudio.yandex.ru/platform/folders/",
"display_name": "Yandex AI Studio",
"can_list_models": true
}
},
{
"id": "cloudflare",
"aliases": [
"cloudflare_ai",
"cf_ai"
],
"protocol": "open_ai_completions",
"api_key_env": "CLOUDFLARE_API_KEY",
"api_key_required": true,
"base_url_env": "CLOUDFLARE_BASE_URL",
"model_env": "CLOUDFLARE_MODEL",
"default_model": "@cf/meta/llama-3.3-70b-instruct-fp8-fast",
"description": "Cloudflare Workers AI",
"setup": {
"kind": "open_ai_compatible",
"secret_name": "llm_cloudflare_api_key",
"display_name": "Cloudflare Workers AI",
"can_list_models": false
}
}
]
+9 -15
View File
@@ -2,36 +2,30 @@
"name": "discord",
"display_name": "Discord Channel",
"kind": "channel",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Talk to your agent in Discord",
"keywords": [
"messaging",
"chat",
"discord",
"bot"
],
"keywords": ["messaging", "chat", "discord", "bot"],
"source": {
"dir": "channels-src/discord",
"capabilities": "discord.capabilities.json",
"crate_name": "discord-channel"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz",
"sha256": "030707431717bca3411a48f311c6ab5f92a45c747de26cafe4f6e3e23a8b3b2d"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "Discord",
"secrets": [
"discord_bot_token"
],
"secrets": ["discord_bot_token"],
"shared_auth": null,
"setup_url": "https://discord.com/developers/applications"
},
"tags": [
"messaging"
]
"tags": ["messaging"]
}
+9 -17
View File
@@ -2,38 +2,30 @@
"name": "slack",
"display_name": "Slack Channel",
"kind": "channel",
"version": "0.2.1",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Talk to your agent in Slack",
"keywords": [
"messaging",
"chat",
"workspace",
"slack"
],
"keywords": ["messaging", "chat", "workspace", "slack"],
"source": {
"dir": "channels-src/slack",
"capabilities": "slack.capabilities.json",
"crate_name": "slack-channel"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
"sha256": "6ed36077b67ac70a041f06f760f93ba79b33269885413c3c3f2c8c87ee60807e"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "Slack",
"secrets": [
"slack_bot_token",
"slack_signing_secret"
],
"secrets": ["slack_bot_token", "slack_signing_secret"],
"shared_auth": null,
"setup_url": "https://api.slack.com/apps"
},
"tags": [
"default",
"messaging"
]
"tags": ["default", "messaging"]
}
+9 -16
View File
@@ -2,37 +2,30 @@
"name": "telegram",
"display_name": "Telegram Channel",
"kind": "channel",
"version": "0.2.2",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Talk to your agent through a Telegram bot",
"keywords": [
"messaging",
"bot",
"chat",
"telegram"
],
"keywords": ["messaging", "bot", "chat", "telegram"],
"source": {
"dir": "channels-src/telegram",
"capabilities": "telegram.capabilities.json",
"crate_name": "telegram-channel"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
"sha256": "98c86895a9c4b0a1e19fe8a47f1ccbfe7e972e112b05e584bc897130dc32283a"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "Telegram",
"secrets": [
"telegram_bot_token"
],
"secrets": ["telegram_bot_token"],
"shared_auth": null,
"setup_url": "https://t.me/BotFather"
},
"tags": [
"default",
"messaging"
]
"tags": ["default", "messaging"]
}
+9 -16
View File
@@ -2,37 +2,30 @@
"name": "whatsapp",
"display_name": "WhatsApp Channel",
"kind": "channel",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Talk to your agent through WhatsApp",
"keywords": [
"messaging",
"chat",
"whatsapp",
"meta"
],
"keywords": ["messaging", "chat", "whatsapp", "meta"],
"source": {
"dir": "channels-src/whatsapp",
"capabilities": "whatsapp.capabilities.json",
"crate_name": "whatsapp-channel"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz",
"sha256": "bd35cad18d87292ea8d2f52db9b514ed9f814a414de910f59073d475c26c4c14"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "Meta",
"secrets": [
"whatsapp_access_token",
"whatsapp_verify_token"
],
"secrets": ["whatsapp_access_token", "whatsapp_verify_token"],
"shared_auth": null,
"setup_url": "https://developers.facebook.com/apps/"
},
"tags": [
"messaging"
]
"tags": ["messaging"]
}
+9 -17
View File
@@ -2,38 +2,30 @@
"name": "github",
"display_name": "GitHub",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "GitHub integration for issues, PRs, repos, and code search",
"keywords": [
"git",
"code",
"issues",
"pull-requests",
"repositories"
],
"keywords": ["git", "code", "issues", "pull-requests", "repositories"],
"source": {
"dir": "tools-src/github",
"capabilities": "github-tool.capabilities.json",
"crate_name": "github-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz",
"sha256": "6fcd32719a4ff15641a4b50fff8984686550f0c491dce60518f4126857d0c544"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "GitHub",
"secrets": [
"github_token"
],
"secrets": ["github_token"],
"shared_auth": null,
"setup_url": "https://github.com/settings/tokens"
},
"tags": [
"default",
"development"
]
"tags": ["default", "development"]
}
+9 -17
View File
@@ -2,38 +2,30 @@
"name": "gmail",
"display_name": "Gmail",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Read, send, and manage Gmail messages and threads",
"keywords": [
"email",
"google",
"mail",
"messaging"
],
"keywords": ["email", "google", "mail", "messaging"],
"source": {
"dir": "tools-src/gmail",
"capabilities": "gmail-tool.capabilities.json",
"crate_name": "gmail-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz",
"sha256": "023da7000b17568bf0e64b2e5013c8a042b2f323c85f1632339231c73d500e39"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"default",
"google",
"messaging"
]
"tags": ["default", "google", "messaging"]
}
+9 -17
View File
@@ -2,38 +2,30 @@
"name": "google-calendar",
"display_name": "Google Calendar",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Create, read, update, and delete Google Calendar events",
"keywords": [
"calendar",
"google",
"scheduling",
"events"
],
"keywords": ["calendar", "google", "scheduling", "events"],
"source": {
"dir": "tools-src/google-calendar",
"capabilities": "google-calendar-tool.capabilities.json",
"crate_name": "google-calendar-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz",
"sha256": "fc42277b65881d6e9bcc5403dc54c7f5b3ddeaaaf04617fce2c5da05d76325f0"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"default",
"google",
"productivity"
]
"tags": ["default", "google", "productivity"]
}
+9 -16
View File
@@ -2,37 +2,30 @@
"name": "google-docs",
"display_name": "Google Docs",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Create and edit Google Docs documents",
"keywords": [
"documents",
"google",
"writing",
"docs"
],
"keywords": ["documents", "google", "writing", "docs"],
"source": {
"dir": "tools-src/google-docs",
"capabilities": "google-docs-tool.capabilities.json",
"crate_name": "google-docs-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz",
"sha256": "385c04abd1e6b8011ccc330e1f4bd7ce58577e488959b51594aa04eb26cbe7cc"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"google",
"productivity"
]
"tags": ["google", "productivity"]
}
+9 -17
View File
@@ -2,38 +2,30 @@
"name": "google-drive",
"display_name": "Google Drive",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Upload, download, search, and manage Google Drive files and folders",
"keywords": [
"storage",
"google",
"files",
"drive"
],
"keywords": ["storage", "google", "files", "drive"],
"source": {
"dir": "tools-src/google-drive",
"capabilities": "google-drive-tool.capabilities.json",
"crate_name": "google-drive-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz",
"sha256": "1b107d575a5d52cc8c76d9a681802190f4373fb485f7f54f445533f097fa37c0"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"default",
"google",
"storage"
]
"tags": ["default", "google", "storage"]
}
+9 -16
View File
@@ -2,37 +2,30 @@
"name": "google-sheets",
"display_name": "Google Sheets",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Read and write Google Sheets spreadsheet data",
"keywords": [
"spreadsheets",
"google",
"data",
"sheets"
],
"keywords": ["spreadsheets", "google", "data", "sheets"],
"source": {
"dir": "tools-src/google-sheets",
"capabilities": "google-sheets-tool.capabilities.json",
"crate_name": "google-sheets-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz",
"sha256": "c4f6b1e8c5126ac2c8a4b98e4283a3afa32223d2488fc3c3a609758c0c9beb90"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"google",
"productivity"
]
"tags": ["google", "productivity"]
}
+9 -15
View File
@@ -2,36 +2,30 @@
"name": "google-slides",
"display_name": "Google Slides",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Create and edit Google Slides presentations",
"keywords": [
"presentations",
"google",
"slides"
],
"keywords": ["presentations", "google", "slides"],
"source": {
"dir": "tools-src/google-slides",
"capabilities": "google-slides-tool.capabilities.json",
"crate_name": "google-slides-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz",
"sha256": "7110b8565340c888e51f99e9c013bf4de8f8a7f7b33bace00eb8fc47831ff20b"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Google",
"secrets": [
"google_oauth_token"
],
"secrets": ["google_oauth_token"],
"shared_auth": "google_oauth_token",
"setup_url": "https://console.cloud.google.com/apis/credentials"
},
"tags": [
"google",
"productivity"
]
"tags": ["google", "productivity"]
}
+31
View File
@@ -0,0 +1,31 @@
{
"name": "okta",
"display_name": "Okta",
"kind": "tool",
"version": "0.1.0",
"description": "Okta SSO for user profile, app catalog, and SSO launch links",
"keywords": ["sso", "identity", "authentication", "okta"],
"source": {
"dir": "tools-src/okta",
"capabilities": "okta-tool.capabilities.json",
"crate_name": "okta-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/okta-wasm32-wasip2.tar.gz",
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Okta",
"secrets": ["okta_oauth_token"],
"shared_auth": null,
"setup_url": "https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/"
},
"tags": ["identity"]
}
+9 -15
View File
@@ -2,36 +2,30 @@
"name": "slack-tool",
"display_name": "Slack Tool",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Your agent uses Slack to post and read messages in your workspace",
"keywords": [
"messaging",
"chat",
"workspace"
],
"keywords": ["messaging", "chat", "workspace"],
"source": {
"dir": "tools-src/slack",
"capabilities": "slack-tool.capabilities.json",
"crate_name": "slack-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-tool-wasm32-wasip2.tar.gz",
"sha256": "6ed36077b67ac70a041f06f760f93ba79b33269885413c3c3f2c8c87ee60807e"
"sha256": null
}
},
"auth_summary": {
"method": "oauth",
"provider": "Slack",
"secrets": [
"slack_bot_token"
],
"secrets": ["slack_bot_token"],
"shared_auth": null,
"setup_url": "https://api.slack.com/apps"
},
"tags": [
"default",
"messaging"
]
"tags": ["default", "messaging"]
}
+9 -16
View File
@@ -2,37 +2,30 @@
"name": "telegram-mtproto",
"display_name": "Telegram Tool",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"version": "0.1.0",
"description": "Your agent uses your Telegram account to read and send messages",
"keywords": [
"messaging",
"chat",
"telegram",
"mtproto"
],
"keywords": ["messaging", "chat", "telegram", "mtproto"],
"source": {
"dir": "tools-src/telegram",
"capabilities": "telegram-tool.capabilities.json",
"crate_name": "telegram-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-mtproto-wasm32-wasip2.tar.gz",
"sha256": "98c86895a9c4b0a1e19fe8a47f1ccbfe7e972e112b05e584bc897130dc32283a"
"sha256": null
}
},
"auth_summary": {
"method": "manual",
"provider": "Telegram",
"secrets": [
"telegram_api_id",
"telegram_api_hash"
],
"secrets": ["telegram_api_id", "telegram_api_hash"],
"shared_auth": null,
"setup_url": "https://my.telegram.org/apps"
},
"tags": [
"messaging"
]
"tags": ["messaging"]
}
-38
View File
@@ -1,38 +0,0 @@
{
"name": "web-search",
"display_name": "Web Search",
"kind": "tool",
"version": "0.2.0",
"wit_version": "0.3.0",
"description": "Search the web using Brave Search API",
"keywords": [
"search",
"web",
"brave",
"internet"
],
"source": {
"dir": "tools-src/web-search",
"capabilities": "web-search-tool.capabilities.json",
"crate_name": "web-search-tool"
},
"artifacts": {
"wasm32-wasip2": {
"url": "https://github.com/nearai/ironclaw/releases/latest/download/web-search-wasm32-wasip2.tar.gz",
"sha256": "66cb2b9b00652385e9f30f17c74902b9222c17c53e9d3bd1ef42f5cab705bcf6"
}
},
"auth_summary": {
"method": "manual",
"provider": "Brave",
"secrets": [
"brave_api_key"
],
"shared_auth": null,
"setup_url": "https://brave.com/search/api/"
},
"tags": [
"default",
"search"
]
}
-74
View File
@@ -1,74 +0,0 @@
#!/usr/bin/env bash
# Build all WASM tools and channels from source.
#
# Verifies that every tool/channel in the registry compiles against the
# current WIT definitions. Used by CI and can be run locally.
#
# Prerequisites:
# rustup target add wasm32-wasip2
# cargo install cargo-component --locked
#
# Usage:
# ./scripts/build-wasm-extensions.sh # build all
# ./scripts/build-wasm-extensions.sh --tools # tools only
# ./scripts/build-wasm-extensions.sh --channels # channels only
set -euo pipefail
cd "$(dirname "$0")/.."
BUILD_TOOLS=true
BUILD_CHANNELS=true
FAILED=()
if [[ "${1:-}" == "--tools" ]]; then
BUILD_CHANNELS=false
elif [[ "${1:-}" == "--channels" ]]; then
BUILD_TOOLS=false
fi
build_extension() {
local manifest_path="$1"
local source_dir
local crate_name
source_dir=$(jq -r '.source.dir' "$manifest_path")
crate_name=$(jq -r '.source.crate_name' "$manifest_path")
local name
name=$(basename "$manifest_path" .json)
if [ ! -d "$source_dir" ]; then
echo " SKIP $name (source dir $source_dir not found)"
return 0
fi
echo " BUILD $name ($crate_name) from $source_dir"
if ! cargo component build --release --manifest-path "$source_dir/Cargo.toml" 2>&1; then
echo " FAIL $name"
FAILED+=("$name")
return 1
fi
echo " OK $name"
}
if $BUILD_TOOLS; then
echo "Building WASM tools..."
for manifest in registry/tools/*.json; do
build_extension "$manifest" || true
done
fi
if $BUILD_CHANNELS; then
echo "Building WASM channels..."
for manifest in registry/channels/*.json; do
build_extension "$manifest" || true
done
fi
echo ""
if [ ${#FAILED[@]} -gt 0 ]; then
echo "FAILED: ${FAILED[*]}"
exit 1
else
echo "All WASM extensions built successfully."
fi
-223
View File
@@ -1,223 +0,0 @@
#!/usr/bin/env bash
# Architecture boundary checks for IronClaw.
# Run as: bash scripts/check-boundaries.sh
# Returns non-zero if hard violations are found.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$REPO_ROOT"
violations=0
echo "=== Architecture Boundary Checks ==="
echo
# --------------------------------------------------------------------------
# Check 1: Direct database driver usage outside the db layer
# --------------------------------------------------------------------------
# tokio_postgres:: and libsql:: types should only appear in:
# - src/db/ (the database abstraction layer)
# - src/workspace/repository.rs (workspace's own DB layer)
# - src/error.rs (needs From impls for driver error types)
# - src/app.rs (bootstraps/initialises the database)
# - src/testing.rs (test infrastructure)
# - src/cli/ (CLI commands that bootstrap DB connections)
# - src/setup/ (onboarding wizard bootstraps DB)
# - src/main.rs (entry point)
#
# Everything else is a boundary violation -- those modules should go through
# the Database trait, not touch driver types directly.
# --------------------------------------------------------------------------
echo "--- Check 1: Direct database driver usage outside db layer ---"
results=$(grep -rn 'tokio_postgres::\|libsql::' src/ \
--include='*.rs' \
| grep -v 'src/db/' \
| grep -v 'src/workspace/repository.rs' \
| grep -v 'src/error.rs' \
| grep -v 'src/app.rs' \
| grep -v 'src/testing.rs' \
| grep -v 'src/cli/' \
| grep -v 'src/setup/' \
| grep -v 'src/main.rs' \
| grep -v '^\s*//' \
| grep -v '//.*tokio_postgres\|//.*libsql' \
|| true)
if [ -n "$results" ]; then
echo "VIOLATION: Direct database driver usage found outside db layer:"
echo "$results"
echo
count=$(echo "$results" | wc -l | tr -d ' ')
echo "($count occurrence(s) -- these modules should use the Database trait)"
violations=$((violations + 1))
else
echo "OK"
fi
echo
# --------------------------------------------------------------------------
# Check 2: .unwrap() / .expect() in production code (heuristic)
# --------------------------------------------------------------------------
# We cannot perfectly distinguish test vs production code with grep alone
# (test modules span many lines). Instead we:
# 1. Exclude files that are entirely test infrastructure
# 2. Exclude lines that are clearly in test code (assert, #[test], etc.)
# 3. Report a per-file summary so reviewers can focus on the worst files
#
# This is a WARNING, not a hard violation.
# --------------------------------------------------------------------------
echo "--- Check 2: .unwrap() / .expect() in production code ---"
# Collect raw matches excluding obvious test-only files and lines
raw_results=$(grep -rn '\.unwrap()\|\.expect(' src/ \
--include='*.rs' \
| grep -v 'src/main.rs' \
| grep -v 'src/testing.rs' \
| grep -v 'src/setup/' \
|| true)
if [ -n "$raw_results" ]; then
total=$(echo "$raw_results" | wc -l | tr -d ' ')
echo "WARNING: ~$total .unwrap()/.expect() calls found in src/ (excluding main/testing/setup)."
echo "Many are in test modules; a per-file breakdown helps triage:"
echo
# Show per-file counts, sorted by count descending, top 15
file_counts=$(echo "$raw_results" | cut -d: -f1 | sort | uniq -c | sort -rn)
echo "$file_counts" | head -15
fc_total=$(echo "$file_counts" | wc -l | tr -d ' ')
if [ "$fc_total" -gt 15 ]; then
echo " ... and $((fc_total - 15)) more files"
fi
echo
echo "(This is a warning for gradual cleanup, not a blocking violation.)"
echo "(Many of these are inside #[cfg(test)] modules which is acceptable.)"
else
echo "OK"
fi
echo
# --------------------------------------------------------------------------
# Check 3: std::env::var reads outside config/bootstrap layers
# --------------------------------------------------------------------------
# Sensitive values should come through Config or the secrets module.
# Direct std::env::var / env::var() reads are allowed in:
# - src/config/ (the config layer itself)
# - src/main.rs (entry point)
# - src/setup/ (onboarding wizard)
# - src/testing.rs (test infrastructure)
# - src/cli/ (CLI commands that read env for bootstrap)
# - src/bootstrap.rs (bootstrap logic)
# --------------------------------------------------------------------------
echo "--- Check 3: Direct env var reads outside config layer ---"
results=$(grep -rn 'std::env::var\|env::var(' src/ \
--include='*.rs' \
| grep -v 'src/config/' \
| grep -v 'src/main.rs' \
| grep -v 'src/setup/' \
| grep -v 'src/testing.rs' \
| grep -v 'src/cli/' \
| grep -v 'src/bootstrap.rs' \
| grep -v '#\[cfg(test)\]' \
| grep -v '#\[test\]' \
| grep -v 'mod tests' \
| grep -v 'fn test_' \
| grep -v '//.*env::var' \
|| true)
if [ -n "$results" ]; then
count=$(echo "$results" | wc -l | tr -d ' ')
echo "WARNING: Direct env var reads found outside config layer ($count occurrences):"
echo "$results"
echo
echo "(Review these -- secrets/config should come through Config or the secrets module)"
else
echo "OK"
fi
echo
# --------------------------------------------------------------------------
# Check 4: Test tier gating — integration tests must use feature flags
# --------------------------------------------------------------------------
# Files in tests/ that connect to PostgreSQL or use DATABASE_URL must be
# gated behind #![cfg(all(feature = "postgres", feature = "integration"))].
# This ensures `cargo test` (no flags) never requires external services.
#
# Heuristic: any test file referencing DATABASE_URL, connect(), PgPool,
# or tokio_postgres should have the cfg gate on the first few lines.
# --------------------------------------------------------------------------
echo "--- Check 4: Test tier gating for integration tests ---"
tier_violations=()
for test_file in tests/*.rs; do
[ -f "$test_file" ] || continue
# Check if the file actually connects to a database (imports DB types
# or calls pool/connect). Mere string references like "DATABASE_URL"
# in config tests don't count.
needs_gate=false
if grep -q 'PgPool\|tokio_postgres::\|create_pool\|\.connect(' "$test_file" 2>/dev/null; then
needs_gate=true
fi
if [ "$needs_gate" = true ]; then
# Check first 5 lines for the cfg gate
if ! head -5 "$test_file" | grep -q 'cfg.*feature.*integration' 2>/dev/null; then
tier_violations+=(" $test_file: needs '#![cfg(all(feature = \"postgres\", feature = \"integration\"))]'")
fi
fi
done
if [ ${#tier_violations[@]} -gt 0 ]; then
echo "VIOLATION: Integration tests missing feature gate:"
printf '%s\n' "${tier_violations[@]}"
echo
echo "(Tests requiring external services must be gated behind the 'integration' feature)"
violations=$((violations + 1))
else
echo "OK"
fi
echo
# --------------------------------------------------------------------------
# Check 5: No silent test-skip patterns (try_connect, is_available, etc.)
# --------------------------------------------------------------------------
# Tests must fail loudly when prerequisites are missing, not silently skip.
# The correct approach is feature-flag gating (#![cfg(feature = "integration")]).
# Patterns like try_connect().is_none() { return; } hide broken tests.
# --------------------------------------------------------------------------
echo "--- Check 5: No silent test-skip patterns ---"
skip_results=$(grep -rn 'try_connect\|is_available.*return\|is_none.*return\|is_err.*return.*//.*skip' tests/ \
--include='*.rs' \
|| true)
if [ -n "$skip_results" ]; then
echo "VIOLATION: Silent test-skip patterns found (use feature gates instead):"
echo "$skip_results"
echo
violations=$((violations + 1))
else
echo "OK"
fi
echo
# --------------------------------------------------------------------------
# Summary
# --------------------------------------------------------------------------
echo "=== Summary ==="
if [ "$violations" -gt 0 ]; then
echo "FAILED: $violations hard violation(s) found"
exit 1
else
echo "PASSED: No hard violations found (review warnings above)"
exit 0
fi
-251
View File
@@ -1,251 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
# CI script: check that version bumps accompany WIT or extension source changes.
# Exit 0 if all checks pass, exit 1 if any version wasn't bumped.
ERRORS=0
# --- Skip mechanism -----------------------------------------------------------
if [[ "${PR_LABELS:-}" == *"skip-version-check"* ]]; then
echo "skip-version-check label detected — skipping all version checks."
exit 0
fi
# Check commit messages for [skip-version-check]
if git log "origin/${GITHUB_BASE_REF:-main}...HEAD" --pretty=format:"%s %b" 2>/dev/null \
| grep -qF '[skip-version-check]'; then
echo "[skip-version-check] found in commit message — skipping all version checks."
exit 0
fi
# --- Determine base branch and changed files ----------------------------------
BASE_BRANCH="${GITHUB_BASE_REF:-main}"
echo "Base branch: $BASE_BRANCH"
# Ensure the base branch ref is available
if ! git rev-parse "origin/${BASE_BRANCH}" >/dev/null 2>&1; then
echo "Fetching origin/${BASE_BRANCH}..."
git fetch origin "$BASE_BRANCH" --depth=1
fi
CHANGED_FILES=$(git diff --name-only "origin/${BASE_BRANCH}...HEAD")
if [[ -z "$CHANGED_FILES" ]]; then
echo "No changed files detected. Nothing to check."
exit 0
fi
# --- Helper functions ---------------------------------------------------------
# Extract the version from a WIT package line like: package near:[email protected];
extract_wit_version() {
local file="$1"
if [[ ! -f "$file" ]]; then
echo ""
return
fi
sed -n 's/^[[:space:]]*package[[:space:]]\+[^@]*@\([0-9][0-9.]*[0-9]\)[[:space:]]*;.*/\1/p' "$file" \
| head -n1
}
# Extract version from the base branch copy of a file
extract_wit_version_base() {
local file="$1"
git show "origin/${BASE_BRANCH}:${file}" 2>/dev/null \
| sed -n 's/^[[:space:]]*package[[:space:]]\+[^@]*@\([0-9][0-9.]*[0-9]\)[[:space:]]*;.*/\1/p' \
| head -n1 || true
}
# Extract a Rust string constant value: pub const NAME: &str = "value";
extract_rust_const() {
local file="$1"
local const_name="$2"
if [[ ! -f "$file" ]]; then
echo ""
return
fi
sed -n "s/^.*${const_name}[[:space:]]*:[[:space:]]*&str[[:space:]]*=[[:space:]]*\"\([^\"]*\)\".*/\1/p" "$file" \
| head -n1
}
# Extract JSON "version" field using jq
extract_json_version() {
local file="$1"
if [[ ! -f "$file" ]]; then
echo ""
return
fi
jq -r '.version // empty' "$file" 2>/dev/null || true
}
# Extract JSON "version" from the base branch copy of a file
extract_json_version_base() {
local file="$1"
git show "origin/${BASE_BRANCH}:${file}" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true
}
# Return 0 if $1 (new) is strictly greater than $2 (old) via sort -V, or old is empty.
version_was_bumped() {
local new="$1"
local old="$2"
if [[ -z "$old" ]]; then
# No prior version — treat as new, no bump required
return 0
fi
if [[ -z "$new" ]]; then
# Version was removed — that's a problem
return 1
fi
if [[ "$new" == "$old" ]]; then
return 1
fi
# Check new > old via sort -V
local highest
highest=$(printf '%s\n%s\n' "$new" "$old" | sort -V | tail -n1)
[[ "$highest" == "$new" ]]
}
# --- 1. WIT changes ----------------------------------------------------------
WIT_TOOL_CHANGED=false
WIT_CHANNEL_CHANGED=false
if echo "$CHANGED_FILES" | grep -qx 'wit/tool\.wit'; then
WIT_TOOL_CHANGED=true
fi
if echo "$CHANGED_FILES" | grep -qx 'wit/channel\.wit'; then
WIT_CHANNEL_CHANGED=true
fi
if $WIT_TOOL_CHANGED; then
echo ""
echo "=== wit/tool.wit changed ==="
NEW_VER=$(extract_wit_version "wit/tool.wit")
OLD_VER=$(extract_wit_version_base "wit/tool.wit")
echo " WIT package version: ${OLD_VER:-<none>} -> ${NEW_VER:-<missing>}"
if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then
echo " ERROR: wit/tool.wit package version was not bumped (${OLD_VER} -> ${NEW_VER:-<missing>})."
ERRORS=$((ERRORS + 1))
else
echo " OK: WIT package version bumped."
fi
# Check WIT_TOOL_VERSION constant matches
CONST_VER=$(extract_rust_const "src/tools/wasm/mod.rs" "WIT_TOOL_VERSION")
if [[ -n "$NEW_VER" && "$CONST_VER" != "$NEW_VER" ]]; then
echo " ERROR: WIT_TOOL_VERSION in src/tools/wasm/mod.rs is '${CONST_VER}' but wit/tool.wit has '${NEW_VER}'. They must match."
ERRORS=$((ERRORS + 1))
elif [[ -n "$NEW_VER" ]]; then
echo " OK: WIT_TOOL_VERSION matches wit/tool.wit."
fi
fi
if $WIT_CHANNEL_CHANGED; then
echo ""
echo "=== wit/channel.wit changed ==="
NEW_VER=$(extract_wit_version "wit/channel.wit")
OLD_VER=$(extract_wit_version_base "wit/channel.wit")
echo " WIT package version: ${OLD_VER:-<none>} -> ${NEW_VER:-<missing>}"
if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then
echo " ERROR: wit/channel.wit package version was not bumped (${OLD_VER} -> ${NEW_VER:-<missing>})."
ERRORS=$((ERRORS + 1))
else
echo " OK: WIT package version bumped."
fi
# Check WIT_CHANNEL_VERSION constant matches
CONST_VER=$(extract_rust_const "src/tools/wasm/mod.rs" "WIT_CHANNEL_VERSION")
if [[ -n "$NEW_VER" && "$CONST_VER" != "$NEW_VER" ]]; then
echo " ERROR: WIT_CHANNEL_VERSION in src/tools/wasm/mod.rs is '${CONST_VER}' but wit/channel.wit has '${NEW_VER}'. They must match."
ERRORS=$((ERRORS + 1))
elif [[ -n "$NEW_VER" ]]; then
echo " OK: WIT_CHANNEL_VERSION matches wit/channel.wit."
fi
fi
if $WIT_TOOL_CHANGED || $WIT_CHANNEL_CHANGED; then
echo ""
echo " WARNING: WIT interface changed. All published registry extensions should bump their versions for compatibility."
fi
# --- 2. Tool source changes ---------------------------------------------------
TOOL_NAMES=$(echo "$CHANGED_FILES" | sed -n 's|^tools-src/\([^/]*\)/.*|\1|p' | sort -u)
if [[ -n "$TOOL_NAMES" ]]; then
echo ""
echo "=== Tool source changes ==="
fi
for tool in $TOOL_NAMES; do
REGISTRY_FILE="registry/tools/${tool}.json"
echo ""
echo " --- tools-src/${tool}/ changed ---"
if [[ ! -f "$REGISTRY_FILE" ]]; then
echo " SKIP: ${REGISTRY_FILE} does not exist yet (new extension?)."
continue
fi
NEW_VER=$(extract_json_version "$REGISTRY_FILE")
OLD_VER=$(extract_json_version_base "$REGISTRY_FILE")
echo " Registry version: ${OLD_VER:-<none>} -> ${NEW_VER:-<missing>}"
if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then
echo " ERROR: ${REGISTRY_FILE} version was not bumped (${OLD_VER} -> ${NEW_VER:-<missing>}). Bump the version when changing tools-src/${tool}/."
ERRORS=$((ERRORS + 1))
else
echo " OK: version bumped."
fi
done
# --- 3. Channel source changes ------------------------------------------------
CHANNEL_NAMES=$(echo "$CHANGED_FILES" | sed -n 's|^channels-src/\([^/]*\)/.*|\1|p' | sort -u)
if [[ -n "$CHANNEL_NAMES" ]]; then
echo ""
echo "=== Channel source changes ==="
fi
for channel in $CHANNEL_NAMES; do
REGISTRY_FILE="registry/channels/${channel}.json"
echo ""
echo " --- channels-src/${channel}/ changed ---"
if [[ ! -f "$REGISTRY_FILE" ]]; then
echo " SKIP: ${REGISTRY_FILE} does not exist yet (new extension?)."
continue
fi
NEW_VER=$(extract_json_version "$REGISTRY_FILE")
OLD_VER=$(extract_json_version_base "$REGISTRY_FILE")
echo " Registry version: ${OLD_VER:-<none>} -> ${NEW_VER:-<missing>}"
if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then
echo " ERROR: ${REGISTRY_FILE} version was not bumped (${OLD_VER} -> ${NEW_VER:-<missing>}). Bump the version when changing channels-src/${channel}/."
ERRORS=$((ERRORS + 1))
else
echo " OK: version bumped."
fi
done
# --- Summary ------------------------------------------------------------------
echo ""
if [[ $ERRORS -gt 0 ]]; then
echo "FAILED: ${ERRORS} version check(s) did not pass. See errors above."
exit 1
else
echo "All version checks passed."
exit 0
fi
-81
View File
@@ -1,81 +0,0 @@
#!/usr/bin/env bash
# commit-msg hook: require regression tests for fix commits.
#
# Installed by scripts/dev-setup.sh as .git/hooks/commit-msg.
# Bypass with [skip-regression-check] in the commit message.
set -euo pipefail
MSG_FILE="$1"
FIRST_LINE=$(head -1 "$MSG_FILE")
# --- 1. Is this a fix commit? ---
if ! grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$FIRST_LINE"; then
exit 0
fi
# --- 2. Skip marker ---
if grep -qF '[skip-regression-check]' "$MSG_FILE"; then
exit 0
fi
# --- 3. Exempt static-only / docs-only changes ---
# Get staged files (commit-msg runs after staging is finalized).
STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMR)
if [ -z "$STAGED_FILES" ]; then
exit 0
fi
ALL_EXEMPT=true
while IFS= read -r file; do
case "$file" in
src/channels/web/static/*) ;;
*.md) ;;
*) ALL_EXEMPT=false; break ;;
esac
done <<< "$STAGED_FILES"
if [ "$ALL_EXEMPT" = true ]; then
exit 0
fi
# --- 4. Look for test changes in staged .rs files ---
# Fast path: new test attributes or test modules in added lines.
if git diff --cached -U0 -- '*.rs' | grep -qE '^\+.*(#\[test\]|#\[tokio::test\]|#\[cfg\(test\)\]|mod tests)'; then
exit 0
fi
# Whole-function context: detect edits inside existing test functions.
# -W shows the full enclosing function, so #[test] appears in context
# lines when changes are inside a test function.
if git diff --cached -W -- '*.rs' | awk '
/^@@/ { if (has_test && has_add) { found=1; exit } has_test=0; has_add=0 }
/^ .*#\[test\]/ || /^ .*#\[tokio::test\]/ || /^ .*#\[cfg\(test\)\]/ || /^ .*mod tests/ { has_test=1 }
/^\+.*#\[test\]/ || /^\+.*#\[tokio::test\]/ || /^\+.*#\[cfg\(test\)\]/ || /^\+.*mod tests/ { has_test=1 }
/^\+[^+]/ { has_add=1 }
END { if (has_test && has_add) found=1; exit !found }
'; then
exit 0
fi
# Also check for new/modified files under tests/
if grep -qE '^tests/' <<< "$STAGED_FILES"; then
exit 0
fi
# --- 5. No test found — block the commit ---
echo ""
echo "╔══════════════════════════════════════════════════════════════╗"
echo "║ REGRESSION TEST REQUIRED ║"
echo "║ ║"
echo "║ This commit looks like a bug fix but has no test changes. ║"
echo "║ Every fix should include a test that reproduces the bug. ║"
echo "║ ║"
echo "║ Options: ║"
echo "║ • Add a #[test] or #[tokio::test] that catches the bug ║"
echo "║ • Add [skip-regression-check] to your commit message ║"
echo "╚══════════════════════════════════════════════════════════════╝"
echo ""
exit 1
-101
View File
@@ -1,101 +0,0 @@
#!/usr/bin/env bash
# Generate an HTML coverage report for a given set of tests.
#
# Usage:
# ./scripts/coverage.sh # all tests (lib only)
# ./scripts/coverage.sh safety # tests matching "safety"
# ./scripts/coverage.sh safety::sanitizer # specific module tests
# ./scripts/coverage.sh test_a test_b test_c # multiple test filters
#
# Options (env vars):
# COV_OPEN=1 Auto-open the report in a browser (default: 1)
# COV_FORMAT=html Output format: html, text, json, lcov (default: html)
# COV_OUT=coverage Output directory (default: coverage/)
# COV_FEATURES="" Extra --features to pass (default: none)
# COV_ALL_TARGETS=0 Set to 1 to include integration tests (default: lib only)
#
# Requires: cargo-llvm-cov (install: cargo install cargo-llvm-cov)
set -euo pipefail
COV_OPEN="${COV_OPEN:-1}"
COV_FORMAT="${COV_FORMAT:-html}"
COV_OUT="${COV_OUT:-coverage}"
COV_FEATURES="${COV_FEATURES:-}"
COV_ALL_TARGETS="${COV_ALL_TARGETS:-0}"
cd "$(git rev-parse --show-toplevel)"
if ! command -v cargo-llvm-cov &>/dev/null; then
echo "ERROR: cargo-llvm-cov not found. Install with: cargo install cargo-llvm-cov"
exit 1
fi
# Clean stale profiling data to avoid "mismatched data" warnings.
cargo llvm-cov clean --workspace 2>/dev/null || true
# Build the cargo llvm-cov command
cmd=(cargo llvm-cov)
# Features
if [[ -n "$COV_FEATURES" ]]; then
cmd+=(--features "$COV_FEATURES")
else
cmd+=(--all-features)
fi
# By default, only run the lib unit tests (fast, no integration test compilation).
# Set COV_ALL_TARGETS=1 to include integration tests.
if [[ "$COV_ALL_TARGETS" != "1" ]]; then
cmd+=(--lib)
fi
# Output format
case "$COV_FORMAT" in
html)
cmd+=(--html --output-dir "$COV_OUT")
;;
text)
cmd+=(--text)
;;
json)
cmd+=(--json --output-path "$COV_OUT/coverage.json")
;;
lcov)
cmd+=(--lcov --output-path "$COV_OUT/lcov.info")
;;
*)
echo "ERROR: Unknown format '$COV_FORMAT'. Use: html, text, json, lcov"
exit 1
;;
esac
# Test name filters (passed after -- to cargo test)
if [[ $# -gt 0 ]]; then
if [[ $# -eq 1 ]]; then
cmd+=(-- "$1")
else
# Join filters with | for regex matching
filter=$(IFS='|'; echo "$*")
cmd+=(-- "$filter")
fi
fi
echo "Running: ${cmd[*]}"
echo ""
"${cmd[@]}"
# Open report
if [[ "$COV_FORMAT" == "html" && "$COV_OPEN" == "1" ]]; then
index="$COV_OUT/html/index.html"
if [[ -f "$index" ]]; then
echo ""
echo "Report: $index"
if command -v open &>/dev/null; then
open "$index"
elif command -v xdg-open &>/dev/null; then
xdg-open "$index"
fi
fi
fi
+5 -19
View File
@@ -24,14 +24,14 @@ if ! command -v rustup &>/dev/null; then
echo "ERROR: rustup not found. Install from https://rustup.rs"
exit 1
fi
echo "[1/6] rustup found: $(rustup --version 2>/dev/null | head -1)"
echo "[1/5] rustup found: $(rustup --version 2>/dev/null | head -1)"
# 2. Add WASM target (required by build.rs for channel compilation)
echo "[2/6] Adding wasm32-wasip2 target..."
echo "[2/5] Adding wasm32-wasip2 target..."
rustup target add wasm32-wasip2
# 3. Install wasm-tools (required by build.rs for WASM component model)
echo "[3/6] Installing wasm-tools..."
echo "[3/5] Installing wasm-tools..."
if command -v wasm-tools &>/dev/null; then
echo " wasm-tools already installed: $(wasm-tools --version)"
else
@@ -39,27 +39,13 @@ else
fi
# 4. Verify the project compiles
echo "[4/6] Running cargo check..."
echo "[4/5] Running cargo check..."
cargo check
# 5. Run tests using libsql temp DB (no Docker/external DB needed)
echo "[5/6] Running tests (no external DB required)..."
echo "[5/5] Running tests (no external DB required)..."
cargo test
# 6. Install git hooks
echo "[6/6] Installing git hooks..."
HOOKS_DIR=$(git rev-parse --git-path hooks 2>/dev/null) || true
if [ -n "$HOOKS_DIR" ]; then
mkdir -p "$HOOKS_DIR"
SCRIPTS_ABS="$(cd "$(dirname "$0")" && pwd)"
ln -sf "$SCRIPTS_ABS/commit-msg-regression.sh" "$HOOKS_DIR/commit-msg"
echo " commit-msg hook installed (regression test enforcement)"
ln -sf "$SCRIPTS_ABS/pre-commit-safety.sh" "$HOOKS_DIR/pre-commit"
echo " pre-commit hook installed (UTF-8, case-sensitivity, /tmp, redaction checks)"
else
echo " Skipped: not a git repository"
fi
echo ""
echo "=== Setup complete ==="
echo ""
-136
View File
@@ -1,136 +0,0 @@
#!/usr/bin/env bash
# Pre-commit safety checks for common issues caught by AI code reviewers.
#
# Can be run standalone: bash scripts/pre-commit-safety.sh
# Or installed as a git pre-commit hook via dev-setup.sh.
#
# Checks staged .rs files for:
# 1. Unsafe UTF-8 byte slicing (panics on multi-byte chars)
# 2. Case-sensitive file extension comparisons
# 3. Hardcoded /tmp paths in tests (flaky in parallel runs)
# 4. Tool parameters logged without redaction (secret leaks)
# 5. Multi-step DB operations without transaction wrapping
#
# Suppress individual lines with an inline "// safety: <reason>" comment.
set -euo pipefail
# Determine a suitable base ref for standalone diffs.
resolve_base_ref() {
local candidates=(
"@{upstream}"
"origin/HEAD"
"origin/main"
"origin/master"
"main"
"master"
)
for ref in "${candidates[@]}"; do
if git rev-parse --verify --quiet "$ref" >/dev/null 2>&1; then
echo "$ref"
return 0
fi
done
echo "pre-commit-safety: could not determine a base Git ref for diff (tried: ${candidates[*]})." >&2
echo "pre-commit-safety: ensure your repository has an upstream or a local main/master branch." >&2
exit 1
}
# Support both pre-commit hook (staged files) and standalone (all changed vs base)
if git diff --cached --quiet 2>/dev/null; then
# No staged changes -- compare working tree against a resolved base ref
BASE_REF="$(resolve_base_ref)"
DIFF_OUTPUT=$(git diff "$BASE_REF" -- '*.rs' 2>/dev/null || true)
else
DIFF_OUTPUT=$(git diff --cached -U0 -- '*.rs' 2>/dev/null || true)
fi
# Early exit if there are no relevant .rs changes
if [ -z "$DIFF_OUTPUT" ]; then
exit 0
fi
WARNINGS=0
warn() {
if [ "$WARNINGS" -eq 0 ]; then
echo ""
echo "=== Pre-commit Safety Checks ==="
echo ""
fi
WARNINGS=$((WARNINGS + 1))
echo " [$1] $2"
}
# 1. Unsafe UTF-8 byte slicing: &s[..N] or &s[..some_var] on strings
# Safe patterns: is_char_boundary, char_indices, // safety:
if echo "$DIFF_OUTPUT" | grep -nE '^\+' | grep -E '\[\.\..*\]' | grep -vE 'is_char_boundary|char_indices|// safety:|as_bytes|Vec<|&\[u8\]|\[u8\]|bytes\(\)|&bytes' | head -3 | grep -q .; then
warn "UTF8" "Possible unsafe byte-index string slicing. Use is_char_boundary() or char_indices()."
echo "$DIFF_OUTPUT" | grep -nE '^\+' | grep -E '\[\.\..*\]' | grep -vE 'is_char_boundary|char_indices|// safety:|as_bytes|Vec<|&\[u8\]|\[u8\]|bytes\(\)|&bytes' | head -3 | sed 's/^/ /'
fi
# 2. Case-sensitive file extension checks
# Match: .ends_with(".png") without prior to_lowercase
if echo "$DIFF_OUTPUT" | grep -nE '^\+.*ends_with\("\.([pP][nN][gG]|[jJ][pP][eE]?[gG]|[gG][iI][fF]|[wW][eE][bB][pP]|[mM][dD])"\)' | grep -vE 'to_lowercase|to_ascii_lowercase|// safety:' | head -3 | grep -q .; then
warn "CASE" "Case-sensitive file extension comparison. Normalize to lowercase first."
echo "$DIFF_OUTPUT" | grep -nE '^\+.*ends_with\("\.([pP][nN][gG]|[jJ][pP][eE]?[gG]|[gG][iI][fF]|[wW][eE][bB][pP]|[mM][dD])"\)' | grep -vE 'to_lowercase|to_ascii_lowercase|// safety:' | head -3 | sed 's/^/ /'
fi
# 3. Hardcoded /tmp paths in test files
if echo "$DIFF_OUTPUT" | grep -nE '^\+.*"/tmp/' | grep -vE 'tempfile|tempdir|// safety:' | head -3 | grep -q .; then
warn "TMPDIR" "Hardcoded /tmp path. Use tempfile::tempdir() for parallel-safe tests."
echo "$DIFF_OUTPUT" | grep -nE '^\+.*"/tmp/' | grep -vE 'tempfile|tempdir|// safety:' | head -3 | sed 's/^/ /'
fi
# 4. Logging tool parameters without redaction
if echo "$DIFF_OUTPUT" | grep -nE '^\+.*tracing::(info|debug|warn|error).*param' | grep -vE 'redact|// safety:' | head -3 | grep -q .; then
warn "REDACT" "Logging tool parameters without redaction. Use redact_params() first."
echo "$DIFF_OUTPUT" | grep -nE '^\+.*tracing::(info|debug|warn|error).*param' | grep -vE 'redact|// safety:' | head -3 | sed 's/^/ /'
fi
# 5. Multi-step DB operations without transaction
# Uses -W (function context) to reduce false positives from existing transactions.
# Suppressible with "// safety:" in the hunk.
DIFF_W_OUTPUT=$(git diff --cached -W -- '*.rs' 2>/dev/null || git diff "$(resolve_base_ref)" -W -- '*.rs' 2>/dev/null || true)
if [ -n "$DIFF_W_OUTPUT" ]; then
HUNK_COUNT=$(echo "$DIFF_W_OUTPUT" | awk '
/^@@/ {
if (count >= 2 && !has_tx && !has_safety) found++
count=0; has_tx=0; has_safety=0
}
/^\+.*\.(execute|query)\(/ { count++ }
/^\+.*(transaction|\.tx\.|\.begin\()/ { has_tx=1 }
/ .*(transaction|\.tx\.|\.begin\()/ { has_tx=1 }
/\/\/ safety:/ { has_safety=1 }
END {
if (count >= 2 && !has_tx && !has_safety) found++
print found+0
}
')
if [ "$HUNK_COUNT" -gt 0 ]; then
warn "TX" "Multiple DB operations in same function without transaction. Wrap in a transaction for atomicity."
echo "$DIFF_W_OUTPUT" | awk '
/^@@/ {
if (count >= 2 && !has_tx && !has_safety) { print buf }
buf=""; count=0; has_tx=0; has_safety=0
}
/^\+.*\.(execute|query)\(/ { count++ }
/^\+.*(transaction|\.tx\.|\.begin\()/ { has_tx=1 }
/ .*(transaction|\.tx\.|\.begin\()/ { has_tx=1 }
/\/\/ safety:/ { has_safety=1 }
{ buf = buf "\n" $0 }
END {
if (count >= 2 && !has_tx && !has_safety) { print buf }
}
' | grep -E '^\+.*\.(execute|query)\(' | head -4 | sed 's/^/ /'
fi
fi
if [ "$WARNINGS" -gt 0 ]; then
echo ""
echo "Found $WARNINGS potential issue(s). Fix them or add '// safety: <reason>' to suppress."
echo ""
exit 1
fi
-225
View File
@@ -1,225 +0,0 @@
---
name: local-test
version: 0.1.0
description: Build, run, and test IronClaw locally using Docker containers and Chrome MCP browser automation.
activation:
keywords:
- test locally
- local test
- docker test
- test my changes
- test in docker
- test web gateway
- spin up test
- test container
patterns:
- "test.*local"
- "docker.*test"
- "spin.*up.*test"
- "test.*changes.*docker"
max_context_tokens: 3000
---
# Local Testing with Docker + Chrome MCP
Use this skill to build, run, and test IronClaw web gateway changes locally using `Dockerfile.test` and Chrome MCP browser automation tools.
## Quick Start
```bash
# Build the test image (libsql-only, no PostgreSQL needed)
docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test .
# Run on port 3003 (default)
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e NEARAI_API_KEY=<key> \
ironclaw-test
# Open in browser
# http://localhost:3003/?token=test
```
## Building the Image
The test Dockerfile uses a two-stage build: Rust compilation with `--features libsql` (no PostgreSQL dependency), then a minimal Debian runtime image.
```bash
docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test .
```
Build takes ~5-10 minutes on first run (cached subsequent builds are faster). The `--platform linux/amd64` flag avoids QEMU warnings on Apple Silicon but can be omitted if targeting native architecture.
## Running Containers
### Required Environment Variables
| Variable | Purpose | Default in Dockerfile |
|----------|---------|----------------------|
| `ONBOARD_COMPLETED=true` | Skip onboarding wizard (exits immediately otherwise) | not set |
| `CLI_ENABLED=false` | Disable TUI/REPL (causes EOF shutdown otherwise) | not set |
### LLM Backend Configuration
Pick ONE of these configurations:
**NEAR AI (API key mode):**
```bash
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e NEARAI_API_KEY=<your-key> \
ironclaw-test
```
**NEAR AI (session token mode):**
```bash
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e NEARAI_SESSION_TOKEN=<sess_xxx> \
-e NEARAI_BASE_URL=https://private.near.ai \
ironclaw-test
```
**OpenAI:**
```bash
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e LLM_BACKEND=openai \
-e OPENAI_API_KEY=<your-key> \
ironclaw-test
```
**Anthropic:**
```bash
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e LLM_BACKEND=anthropic \
-e ANTHROPIC_API_KEY=<your-key> \
ironclaw-test
```
**Dummy run (no LLM, just test the UI loads):**
```bash
docker run --rm -p 3003:3003 \
-e ONBOARD_COMPLETED=true \
-e CLI_ENABLED=false \
-e NEARAI_API_KEY=dummy \
ironclaw-test
```
### Common Overrides
| Variable | Purpose | Example |
|----------|---------|---------|
| `GATEWAY_PORT` | Change the listen port | `3003` (default) |
| `GATEWAY_AUTH_TOKEN` | Auth token for API | `test` (default) |
| `NEARAI_MODEL` | Override LLM model | `claude-3-5-sonnet-20241022` |
| `RUST_LOG` | Logging verbosity | `ironclaw=debug` |
| `ROUTINES_ENABLED` | Enable routines | `true`/`false` |
| `SKILLS_ENABLED` | Enable skills system | `true` (default) |
### Multi-Instance Testing
Run multiple containers on different host ports:
```bash
docker run --rm -d --name ic-test-a -p 3003:3003 -e ONBOARD_COMPLETED=true -e CLI_ENABLED=false -e NEARAI_API_KEY=dummy ironclaw-test
docker run --rm -d --name ic-test-b -p 3004:3003 -e ONBOARD_COMPLETED=true -e CLI_ENABLED=false -e NEARAI_API_KEY=dummy ironclaw-test
```
## Chrome MCP Testing Workflow
Use the Claude for Chrome browser automation tools to test the web UI.
### Step 1: Get Browser Context
```
mcp__claude-in-chrome__tabs_context_mcp
```
Always start here to see current tabs and get fresh tab IDs.
### Step 2: Open the Gateway
```
mcp__claude-in-chrome__tabs_create_mcp url=http://localhost:3003/?token=test
```
### Step 3: Verify the Page
```
mcp__claude-in-chrome__read_page
```
Check for:
- "Connected" indicator in top-right
- All tabs visible: Chat, Memory, Jobs, Routines, Extensions, Skills
### Step 4: Take Screenshots
```
mcp__claude-in-chrome__computer action=screenshot
```
### Step 5: Test Mobile Viewport
```
mcp__claude-in-chrome__resize_window width=375 height=812
mcp__claude-in-chrome__computer action=screenshot
```
Reset to desktop:
```
mcp__claude-in-chrome__resize_window width=1280 height=800
```
### Step 6: Run JavaScript Checks
```
mcp__claude-in-chrome__javascript_tool script="document.querySelector('.connection-status')?.textContent"
```
### Step 7: Test Interactions
Click tabs, send messages, search skills — use `computer` tool with `action=click` and coordinate-based clicks, or use `find` + `form_input` for text entry.
## Cleanup
```bash
# Stop a specific container
docker stop ic-test-a
# Stop all test containers
docker ps --filter ancestor=ironclaw-test -q | xargs -r docker stop
# Remove the test image
docker rmi ironclaw-test
```
## Troubleshooting
### Container exits immediately
- **Missing `ONBOARD_COMPLETED=true`**: The onboarding wizard tries to read stdin, gets EOF, and exits.
- **Missing `CLI_ENABLED=false`**: The REPL channel reads stdin, gets EOF, and shuts down the agent.
### "Model not found" or LLM errors
- Check that your API key/token is valid and the model name is correct.
- For NEAR AI session token mode, you also need `NEARAI_BASE_URL=https://private.near.ai`.
### Platform mismatch warnings on Apple Silicon
- The `--platform linux/amd64` flag causes QEMU emulation warnings — these are harmless.
- Alternatively, omit the flag and build natively if your dependencies support ARM64.
### Port already in use
- The dev server defaults to port 3001; the test Dockerfile defaults to 3003 to avoid conflicts.
- Use a different host port: `-p 3005:3003`.
### Cannot connect from browser
- Verify `GATEWAY_HOST=0.0.0.0` (set by default in Dockerfile).
- Check the container logs: `docker logs <container-id>`.
- Make sure you include the token query param: `?token=test`.
-54
View File
@@ -1,54 +0,0 @@
---
name: review-checklist
version: 0.1.0
description: Pre-merge review checklist based on recurring AI reviewer feedback patterns
activation:
patterns:
- "review.*checklist"
- "ready to merge"
- "pre-merge check"
- "check.*before.*merge"
keywords:
- review
- checklist
- merge
- pre-merge
max_context_tokens: 1500
---
# Pre-Merge Review Checklist
Before merging, verify these items. They represent the most common issues caught by automated code reviewers (Copilot, Gemini) on IronClaw PRs.
## Database Operations
- [ ] Multi-step DB operations are wrapped in transactions (INSERT+INSERT, UPDATE+DELETE, read-modify-write)
- [ ] Both postgres AND libsql backends updated for any new Database trait methods
- [ ] Migrations are atomic (SQL execution + version recording in same transaction)
## Security & Data Safety
- [ ] Tool parameters are redacted via `redact_params()` before logging or SSE/WebSocket broadcast
- [ ] URL validation resolves DNS before checking for private/loopback IPs (anti-SSRF via DNS rebinding)
- [ ] Destructive tools have `requires_approval()` returning `Always` or `UnlessAutoApproved`
- [ ] Data from worker containers is treated as untrusted (tool domain checks, server-side nesting depth)
- [ ] No secrets or credentials in error messages, logs, or SSE events
## String Safety
- [ ] No byte-index slicing (`&s[..n]`) on external/user strings -- use `is_char_boundary()` or `char_indices()`
- [ ] File extension and media type comparisons are case-insensitive (`.to_ascii_lowercase()` before matching)
- [ ] Path comparisons are case-insensitive where needed (macOS/Windows filesystems)
## Trait Wrappers & Decorator Chain
- [ ] New `LlmProvider` trait methods are delegated in ALL wrapper types (grep `impl LlmProvider for`)
- [ ] New trait methods are tested through the full decorator/provider chain, not just the base impl
- [ ] Default trait method implementations are intentional -- wrappers that silently return defaults are bugs
## Tests
- [ ] Temporary files/dirs use `tempfile` crate, no hardcoded `/tmp/` paths
- [ ] Tests don't mutate global statics without synchronization (use per-test state or `serial_test`)
- [ ] Tests don't make real network requests (use mocks, stubs, or RFC 5737 TEST-NET IPs like 192.0.2.1)
- [ ] Test names and comments match actual test behavior and assertions
## Comments & Documentation
- [ ] Code comments match actual behavior (especially route paths, tool names, function semantics)
- [ ] Spec/README files updated if module behavior changed
- [ ] Error messages are clear and non-redundant (don't nest tool name inside tool error that already contains it)
-171
View File
@@ -1,171 +0,0 @@
# Agent Module
Core agent logic. This is the most complex subsystem — read this before working in `src/agent/`.
## Module Map
| File | Role |
|------|------|
| `agent_loop.rs` | `Agent` struct, `AgentDeps`, main `run()` event loop. Delegates to siblings. |
| `dispatcher.rs` | Agentic loop for conversational turns: LLM call → tool execution → repeat. Injects skill context. Returns `Response` or `NeedApproval`. |
| `thread_ops.rs` | Thread/session operations: `process_user_input`, undo/redo, approval, auth-mode interception, DB hydration, compaction. |
| `commands.rs` | System command handlers (`/help`, `/model`, `/status`, `/skills`, etc.) and job intent handlers. |
| `session.rs` | Data model: `Session``Thread``Turn`. State machines for threads and turns. |
| `session_manager.rs` | Lifecycle: create/lookup sessions, map external thread IDs to internal UUIDs, prune stale sessions, manage undo managers. |
| `router.rs` | Routes explicit `/commands` to `MessageIntent`. Natural language bypasses the router entirely. |
| `scheduler.rs` | Parallel job scheduling. Maintains `jobs` map (full LLM-driven) and `subtasks` map (tool-exec/background). |
| `worker.rs` | Per-job execution for background scheduler jobs: calls LLM, runs tools, handles the reasoning loop. Distinct from `dispatcher.rs`. |
| `compaction.rs` | Context window management: summarize old turns, write to workspace daily log, trim context. Three strategies. |
| `context_monitor.rs` | Detects memory pressure. Suggests `CompactionStrategy` based on usage level. |
| `self_repair.rs` | Detects stuck jobs and broken tools, attempts recovery. |
| `heartbeat.rs` | Proactive periodic execution. Reads `HEARTBEAT.md`, notifies via channel if findings. |
| `submission.rs` | Parses all user submissions into typed variants before routing. |
| `undo.rs` | Turn-based undo/redo with checkpoints. Checkpoints store message lists (max 20 by default). |
| `routine.rs` | `Routine` types: `Trigger` (cron/event/webhook/manual) + `RoutineAction` (lightweight/full_job) + `RoutineGuardrails`. |
| `routine_engine.rs` | Cron ticker and event matcher. Fires routines when triggers match. Lightweight runs inline; full_job dispatches to `Scheduler`. |
| `task.rs` | Task types for the scheduler: `Job`, `ToolExec`, `Background`. Used by `spawn_subtask` and `spawn_batch`. |
| `cost_guard.rs` | LLM spend and action-rate enforcement. Tracks daily budget (cents) and hourly call rate. Lives in `AgentDeps`. |
| `job_monitor.rs` | Subscribes to SSE broadcast and injects Claude Code (container) output back into the agent loop as `IncomingMessage`. |
## Session / Thread / Turn Model
```
Session (per user)
└── Thread (per conversation — can have many)
└── Turn (per request/response pair)
├── user_input: String
├── response: Option<String>
├── tool_calls: Vec<ToolCall>
└── state: TurnState (Pending | Running | Complete | Failed)
```
- A session has one **active thread** at a time; threads can be switched.
- Turns are append-only. Undo rolls back by restoring a prior checkpoint (message list, not a full thread snapshot).
- `UndoManager` is per-thread, stored in `SessionManager`, not on `Session` itself. Max 20 checkpoints (oldest dropped when exceeded).
- Group chat detection: if `metadata.chat_type` is `group`/`channel`/`supergroup`, `MEMORY.md` is excluded from the system prompt to prevent leaking personal context.
- **Auth mode**: if a thread has `pending_auth` set (e.g. from `tool_auth` returning `awaiting_token`), the next user message is intercepted before any turn creation, logging, or safety validation and sent directly to the credential store. Any control submission (undo, interrupt, etc.) cancels auth mode.
- `ThreadState` values: `Idle`, `Processing`, `AwaitingApproval`, `Completed`, `Interrupted`.
- `SessionManager` maps `(user_id, channel, external_thread_id)` → internal UUID. Prunes idle sessions every 10 minutes (warns at 1000 sessions).
## Agentic Loop (dispatcher.rs)
The `dispatcher.rs` module handles **direct conversational turns** (user messages processed inline by the main agent). Background scheduler jobs use `worker.rs` instead — these are two separate execution paths.
```
run_agentic_loop() [dispatcher.rs — conversational turns]
1. Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.)
2. Detect group chat from metadata; exclude MEMORY.md if group chat
3. Select active skills (keyword/pattern scoring against message content)
4. Build skill context block (injected before user message)
5. LLM call → text response OR tool calls
6. If tool calls:
a. Check tool approval (session auto-approvals, pending approval queue)
b. Execute tools (parallel via JoinSet)
c. Sanitize results through SafetyLayer
d. Feed results back → goto 5
7. Return AgenticLoopResult::Response or NeedApproval
```
**Tool approval:** Tools flagged `requires_approval` pause the loop and return `NeedApproval`. The web gateway stores the `PendingApproval` in session state and sends an `approval_needed` SSE event. The user's approval/deny resumes the loop.
**worker.rs vs dispatcher.rs:** `dispatcher.rs` runs the agentic loop for user-initiated conversational turns (holds session lock, tracks turns). `worker.rs` is spawned by the `Scheduler` for background jobs created via `CreateJob` / `/job` — it runs independently of the session and has its own LLM reasoning loop with planning support (`use_planning` flag).
## Command Routing (router.rs)
The `Router` handles explicit `/commands` (prefix `/`). It parses them into `MessageIntent` variants: `CreateJob`, `CheckJobStatus`, `CancelJob`, `ListJobs`, `HelpJob`, `Command`. Natural language messages bypass the router entirely — they go directly to `dispatcher.rs` via `process_user_input`. Note: most user-facing commands (undo, compact, etc.) are handled by `SubmissionParser` before the router runs, so `Router` only sees unrecognized `/xxx` patterns that haven't already been claimed by `submission.rs`.
## Compaction
Triggered by `ContextMonitor` when token usage approaches the model's context limit.
**Token estimation**: Word-count × 1.3 + 4 overhead per message. Default context limit: 100,000 tokens. Compaction threshold: 80% (configurable).
Three strategies, chosen by `ContextMonitor.suggest_compaction()` based on usage ratio:
- **MoveToWorkspace** — Writes full turn transcript to workspace daily log, keeps 10 recent turns. Used when usage is 8085% (moderate). Falls back to `Truncate(5)` if no workspace.
- **Summarize** (`keep_recent: N`) — LLM generates a summary of old turns, writes it to workspace daily log (`daily/YYYY-MM-DD.md`), removes old turns. Used when usage is 8595%.
- **Truncate** (`keep_recent: N`) — Removes oldest turns without summarization (fast path). Used when usage >95% (critical).
If the LLM call for summarization fails, the error propagates — turns are **not** truncated on failure.
Manual trigger: user sends `/compact` (parsed by `submission.rs`).
## Scheduler
`Scheduler` maintains two maps under `Arc<RwLock<HashMap>>`:
- `jobs` — full LLM-driven jobs, each with a `Worker` and an `mpsc` channel for `WorkerMessage` (`Start`, `Stop`, `Ping`, `UserMessage`).
- `subtasks` — lightweight `ToolExec` or `Background` tasks spawned via `spawn_subtask()` / `spawn_batch()`.
**Preferred entry point**: `dispatch_job()` — creates context, optionally sets metadata, persists to DB (so FK references from `job_actions`/`llm_calls` are valid immediately), then calls `schedule()`. Don't call `schedule()` directly unless you've already persisted.
Check-insert is done under a single write lock to prevent TOCTOU races. A cleanup task polls every second for job completion and removes the entry from the map.
`spawn_subtask()` returns a `oneshot::Receiver` — callers must await it to get the result. `spawn_batch()` runs all tasks concurrently and returns results in input order.
## Self-Repair
`DefaultSelfRepair` runs on `repair_check_interval` (from `AgentConfig`). It:
1. Calls `ContextManager::find_stuck_jobs()` to find jobs in `JobState::Stuck`.
2. Attempts `ctx.attempt_recovery()` (transitions back to `InProgress`).
3. Returns `ManualRequired` if `repair_attempts >= max_repair_attempts`.
4. Detects broken tools via `store.get_broken_tools(5)` (threshold: 5 failures). Requires `with_store()` to be called; returns empty without a store.
5. Attempts to rebuild broken tools via `SoftwareBuilder`. Requires `with_builder()` to be called; returns `ManualRequired` without a builder.
Note: the `stuck_threshold` duration is stored but currently unused (marked `#[allow(dead_code)]`). Stuck detection relies on `JobState::Stuck` being set by the state machine, not wall-clock time comparison.
Repair results: `Success`, `Retry`, `Failed`, `ManualRequired`. `Retry` does NOT notify the user (to avoid spam).
## Key Invariants
- Never call `.unwrap()` or `.expect()` — use `?` with proper error mapping.
- All state mutations on `Session`/`Thread` happen under `Arc<Mutex<Session>>` lock.
- The agent loop is single-threaded per thread; parallel execution happens at the job/scheduler level.
- Skills are selected **deterministically** (no LLM call) — see `skills/selector.rs`.
- Tool results pass through `SafetyLayer` before returning to LLM (sanitizer → validator → policy → leak detector).
- `SessionManager` uses double-checked locking for session creation. Read lock first (fast path), then write lock with re-check to prevent duplicate sessions.
- `Scheduler.schedule()` holds the write lock for the entire check-insert sequence — don't hold any other locks when calling it.
- `cheap_llm` in `AgentDeps` is used for heartbeat and other lightweight tasks. Falls back to main `llm` if `None`. Use `agent.cheap_llm()` accessor, not `deps.cheap_llm` directly.
- `CostGuard.check_allowed()` must be called **before** LLM calls; `record_llm_call()` must be called **after**. Both calls are separate — the guard does not auto-record.
- `BeforeInbound` and `BeforeOutbound` hooks run for every user message and agent response respectively. Hooks can modify content or reject. Hook errors are logged but **fail-open** (processing continues).
## Complete Submission Command Reference
All commands parsed by `SubmissionParser::parse()`:
| Input | Variant | Notes |
|-------|---------|-------|
| `/undo` | `Undo` | |
| `/redo` | `Redo` | |
| `/interrupt`, `/stop` | `Interrupt` | |
| `/compact` | `Compact` | |
| `/clear` | `Clear` | |
| `/heartbeat` | `Heartbeat` | |
| `/summarize`, `/summary` | `Summarize` | |
| `/suggest` | `Suggest` | |
| `/new`, `/thread new` | `NewThread` | |
| `/thread <uuid>` | `SwitchThread` | Must be valid UUID |
| `/resume <uuid>` | `Resume` | Must be valid UUID |
| `/status [id]`, `/progress [id]`, `/list` | `JobStatus` | `/list` = all jobs |
| `/cancel <id>` | `JobCancel` | |
| `/quit`, `/exit`, `/shutdown` | `Quit` | |
| `yes/y/approve/ok` and aliases | `ApprovalResponse { approved: true, always: false }` | |
| `always/a` and aliases | `ApprovalResponse { approved: true, always: true }` | |
| `no/n/deny/reject/cancel` and aliases | `ApprovalResponse { approved: false }` | |
| JSON `ExecApproval{...}` | `ExecApproval` | From web gateway approval endpoint |
| `/help`, `/?` | `SystemCommand { "help" }` | Bypasses thread-state checks |
| `/version` | `SystemCommand { "version" }` | |
| `/tools` | `SystemCommand { "tools" }` | |
| `/skills [search <q>]` | `SystemCommand { "skills" }` | |
| `/ping` | `SystemCommand { "ping" }` | |
| `/debug` | `SystemCommand { "debug" }` | |
| `/model [name]` | `SystemCommand { "model" }` | |
| Everything else | `UserInput` | Starts a new agentic turn |
**`SystemCommand` vs control**: `SystemCommand` variants bypass thread-state checks entirely (no session lock, no turn creation). `Quit` returns `Ok(None)` from `handle_message` which breaks the main loop.
## Adding a New Submission Command
Submissions are special messages parsed in `submission.rs` before the agentic loop runs. To add a new one:
1. Add a variant to `Submission` enum in `submission.rs`
2. Add parsing in `SubmissionParser::parse()`
3. Handle in `agent_loop.rs` where `SubmissionResult` is matched (the `match submission { ... }` block in `handle_message`)
4. Implement the handler method (usually in `thread_ops.rs` for session operations, or `commands.rs` for system commands)
+5 -140
View File
@@ -73,14 +73,6 @@ pub struct AgentDeps {
pub hooks: Arc<HookRegistry>,
/// Cost enforcement guardrails (daily budget, hourly rate limits).
pub cost_guard: Arc<crate::agent::cost_guard::CostGuard>,
/// SSE broadcast sender for live job event streaming to the web gateway.
pub sse_tx: Option<tokio::sync::broadcast::Sender<crate::channels::web::types::SseEvent>>,
/// HTTP interceptor for trace recording/replay.
pub http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>,
/// Audio transcription middleware for voice messages.
pub transcription: Option<Arc<crate::transcription::TranscriptionMiddleware>>,
/// Document text extraction middleware for PDF, DOCX, PPTX, etc.
pub document_extraction: Option<Arc<crate::document_extraction::DocumentExtractionMiddleware>>,
}
/// The main agent that coordinates all components.
@@ -96,9 +88,6 @@ pub struct Agent {
pub(super) heartbeat_config: Option<HeartbeatConfig>,
pub(super) hygiene_config: Option<crate::config::HygieneConfig>,
pub(super) routine_config: Option<RoutineConfig>,
/// Optional slot to expose the routine engine to the gateway for manual triggering.
pub(super) routine_engine_slot:
Option<Arc<tokio::sync::RwLock<Option<Arc<crate::agent::routine_engine::RoutineEngine>>>>>,
}
impl Agent {
@@ -122,7 +111,7 @@ impl Agent {
let session_manager = session_manager.unwrap_or_else(|| Arc::new(SessionManager::new()));
let mut scheduler = Scheduler::new(
let scheduler = Arc::new(Scheduler::new(
config.clone(),
context_manager.clone(),
deps.llm.clone(),
@@ -130,14 +119,7 @@ impl Agent {
deps.tools.clone(),
deps.store.clone(),
deps.hooks.clone(),
);
if let Some(ref tx) = deps.sse_tx {
scheduler.set_sse_sender(tx.clone());
}
if let Some(ref interceptor) = deps.http_interceptor {
scheduler.set_http_interceptor(Arc::clone(interceptor));
}
let scheduler = Arc::new(scheduler);
));
Self {
config,
@@ -151,18 +133,9 @@ impl Agent {
heartbeat_config,
hygiene_config,
routine_config,
routine_engine_slot: None,
}
}
/// Set the routine engine slot for exposing the engine to the gateway.
pub fn set_routine_engine_slot(
&mut self,
slot: Arc<tokio::sync::RwLock<Option<Arc<crate::agent::routine_engine::RoutineEngine>>>>,
) {
self.routine_engine_slot = Some(slot);
}
// Convenience accessors
/// Get the scheduler (for external wiring, e.g. CreateJobTool).
@@ -354,19 +327,8 @@ impl Agent {
let heartbeat_handle = if let Some(ref hb_config) = self.heartbeat_config {
if hb_config.enabled {
if let Some(workspace) = self.workspace() {
let mut config = AgentHeartbeatConfig::default()
let config = AgentHeartbeatConfig::default()
.with_interval(std::time::Duration::from_secs(hb_config.interval_secs));
config.quiet_hours_start = hb_config.quiet_hours_start;
config.quiet_hours_end = hb_config.quiet_hours_end;
config.timezone = hb_config
.timezone
.clone()
.or_else(|| Some(self.config.default_timezone.clone()));
if let (Some(user), Some(channel)) =
(&hb_config.notify_user, &hb_config.notify_channel)
{
config = config.with_notify(user, channel);
}
// Set up notification channel
let (notify_tx, mut notify_rx) =
@@ -417,8 +379,8 @@ impl Agent {
hygiene,
workspace.clone(),
self.cheap_llm().clone(),
self.safety().clone(),
Some(notify_tx),
self.store().map(Arc::clone),
))
} else {
tracing::warn!("Heartbeat enabled but no workspace available");
@@ -509,11 +471,6 @@ impl Agent {
// SAFETY: self is consumed by run(), we can smuggle the engine in
// via a local to use in the message loop below.
// Expose engine to gateway for manual triggering
if let Some(ref slot) = self.routine_engine_slot {
*slot.write().await = Some(Arc::clone(&engine));
}
tracing::info!(
"Routines enabled: cron ticker every {}s, max {} concurrent",
rt_config.cron_check_interval_secs,
@@ -556,20 +513,6 @@ impl Agent {
}
};
// Apply transcription middleware to audio attachments
let mut message = message;
if let Some(ref transcription) = self.deps.transcription {
transcription.process(&mut message).await;
}
// Apply document extraction middleware to document attachments
if let Some(ref doc_extraction) = self.deps.document_extraction {
doc_extraction.process(&mut message).await;
}
// Store successfully extracted document text in workspace for indexing
self.store_extracted_documents(&message).await;
match self.handle_message(&message).await {
Ok(Some(response)) if !response.is_empty() => {
// Hook: BeforeOutbound — allow hooks to modify or suppress outbound
@@ -668,73 +611,6 @@ impl Agent {
Ok(())
}
/// Store extracted document text in workspace memory for future search/recall.
async fn store_extracted_documents(&self, message: &IncomingMessage) {
let workspace = match self.workspace() {
Some(ws) => ws,
None => return,
};
for attachment in &message.attachments {
if attachment.kind != crate::channels::AttachmentKind::Document {
continue;
}
let text = match &attachment.extracted_text {
Some(t) if !t.starts_with('[') => t, // skip error messages like "[Failed to..."
_ => continue,
};
// Sanitize filename: strip path separators to prevent directory traversal
let raw_name = attachment.filename.as_deref().unwrap_or("unnamed_document");
let filename: String = raw_name
.chars()
.map(|c| {
if c == '/' || c == '\\' || c == '\0' {
'_'
} else {
c
}
})
.collect();
let filename = filename.trim_start_matches('.');
let filename = if filename.is_empty() {
"unnamed_document"
} else {
filename
};
let date = chrono::Utc::now().format("%Y-%m-%d");
let path = format!("documents/{date}/{filename}");
let header = format!(
"# {filename}\n\n\
> Uploaded by **{}** via **{}** on {date}\n\
> MIME: {} | Size: {} bytes\n\n---\n\n",
message.user_id,
message.channel,
attachment.mime_type,
attachment.size_bytes.unwrap_or(0),
);
let content = format!("{header}{text}");
match workspace.write(&path, &content).await {
Ok(_) => {
tracing::info!(
path = %path,
text_len = text.len(),
"Stored extracted document in workspace memory"
);
}
Err(e) => {
tracing::warn!(
path = %path,
error = %e,
"Failed to store extracted document in workspace"
);
}
}
}
}
async fn handle_message(&self, message: &IncomingMessage) -> Result<Option<String>, Error> {
// Set message tool context for this turn (current channel and target)
// For Signal, use signal_target from metadata (group:ID or phone number),
@@ -751,10 +627,6 @@ impl Agent {
// Parse submission type first
let mut submission = SubmissionParser::parse(&message.content);
tracing::debug!(
"[agent_loop] Parsed submission: {:?}",
std::any::type_name_of_val(&submission)
);
// Hook: BeforeInbound — allow hooks to modify or reject user input
if let Submission::UserInput { ref content } = submission {
@@ -839,14 +711,7 @@ impl Agent {
.await
}
Submission::SystemCommand { command, args } => {
tracing::debug!(
"[agent_loop] SystemCommand: command={}, channel={}",
command,
message.channel
);
// Authorization checks (including restart channel check) are enforced in handle_system_command
self.handle_system_command(&command, &args, &message.channel)
.await
self.handle_system_command(&command, &args).await
}
Submission::Undo => self.process_undo(session, thread_id).await,
Submission::Redo => self.process_redo(session, thread_id).await,
-307
View File
@@ -1,307 +0,0 @@
//! Augment user message content with structured attachment context.
use base64::Engine;
use crate::channels::{AttachmentKind, IncomingAttachment};
use crate::llm::{ContentPart, ImageUrl};
/// Result of processing attachments for the LLM pipeline.
pub struct AugmentResult {
/// Augmented text content with attachment metadata appended.
pub text: String,
/// Image content parts to include as multimodal input.
pub image_parts: Vec<ContentPart>,
}
/// Process attachments into augmented text and multimodal image parts.
///
/// Returns `None` if `attachments` is empty (caller should use original content).
/// Returns `Some(AugmentResult)` with:
/// - `text`: original content + `<attachments>` block (metadata, transcripts, etc.)
/// - `image_parts`: `ContentPart::ImageUrl` entries for images with data
pub fn augment_with_attachments(
content: &str,
attachments: &[IncomingAttachment],
) -> Option<AugmentResult> {
if attachments.is_empty() {
return None;
}
let mut text = content.to_string();
text.push_str("\n\n<attachments>");
let mut image_parts = Vec::new();
for (i, att) in attachments.iter().enumerate() {
text.push('\n');
text.push_str(&format_attachment(i + 1, att));
// Build multimodal image part when image data is available
if att.kind == AttachmentKind::Image && !att.data.is_empty() {
let b64 = base64::engine::general_purpose::STANDARD.encode(&att.data);
let data_url = format!("data:{};base64,{}", att.mime_type, b64);
image_parts.push(ContentPart::ImageUrl {
image_url: ImageUrl {
url: data_url,
detail: None,
},
});
}
}
text.push_str("\n</attachments>");
Some(AugmentResult { text, image_parts })
}
/// Escape a string for use as an XML attribute value.
fn escape_xml_attr(s: &str) -> String {
s.replace('&', "&amp;")
.replace('"', "&quot;")
.replace('<', "&lt;")
.replace('>', "&gt;")
}
/// Escape a string for use as XML text content.
fn escape_xml_text(s: &str) -> String {
s.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
}
fn format_attachment(index: usize, att: &IncomingAttachment) -> String {
let filename = escape_xml_attr(att.filename.as_deref().unwrap_or("unknown"));
let mime = escape_xml_attr(&att.mime_type);
match &att.kind {
AttachmentKind::Audio => {
let duration_attr = att
.duration_secs
.map(|d| format!(" duration=\"{d}s\""))
.unwrap_or_default();
let body = match &att.extracted_text {
Some(text) => format!("Transcript: {}", escape_xml_text(text)),
None => "Audio transcript unavailable.".to_string(),
};
format!(
"<attachment index=\"{index}\" type=\"audio\" filename=\"{filename}\"{duration_attr}>\n\
{body}\n\
</attachment>"
)
}
AttachmentKind::Image => {
let size_attr = att
.size_bytes
.map(|s| format!(" size=\"{}\"", format_size(s)))
.unwrap_or_default();
let body = if att.data.is_empty() {
"[Image attached — visual content not available in this conversation]"
} else {
"[Image attached — sent as visual content]"
};
format!(
"<attachment index=\"{index}\" type=\"image\" filename=\"{filename}\" mime=\"{mime}\"{size_attr}>\n\
{body}\n\
</attachment>"
)
}
AttachmentKind::Document => {
let body: String = match &att.extracted_text {
Some(text) => escape_xml_text(text),
None => {
let size_info = att
.size_bytes
.map(|s| format!(" size=\"{}\"", format_size(s)))
.unwrap_or_default();
return format!(
"<attachment index=\"{index}\" type=\"document\" filename=\"{filename}\" mime=\"{mime}\"{size_info}>\n\
[Document attached text extraction unavailable]\n\
</attachment>"
);
}
};
let size_attr = att
.size_bytes
.map(|s| format!(" size=\"{}\"", format_size(s)))
.unwrap_or_default();
format!(
"<attachment index=\"{index}\" type=\"document\" filename=\"{filename}\" mime=\"{mime}\"{size_attr}>\n\
{body}\n\
</attachment>"
)
}
}
}
fn format_size(bytes: u64) -> String {
if bytes < 1024 {
format!("{bytes}B")
} else if bytes < 1024 * 1024 {
format!("{}KB", bytes / 1024)
} else {
format!("{:.1}MB", bytes as f64 / (1024.0 * 1024.0))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_attachment(kind: AttachmentKind) -> IncomingAttachment {
IncomingAttachment {
id: "test-id".to_string(),
kind,
mime_type: "application/octet-stream".to_string(),
filename: None,
size_bytes: None,
source_url: None,
storage_key: None,
extracted_text: None,
data: vec![],
duration_secs: None,
}
}
#[test]
fn empty_attachments_returns_none() {
assert!(augment_with_attachments("hello", &[]).is_none());
}
#[test]
fn audio_with_transcript() {
let mut att = make_attachment(AttachmentKind::Audio);
att.filename = Some("voice.ogg".to_string());
att.extracted_text = Some("Hello, can you help me?".to_string());
att.duration_secs = Some(5);
let result = augment_with_attachments("hi", &[att]).unwrap();
assert!(result.text.starts_with("hi\n\n<attachments>"));
assert!(result.text.contains("type=\"audio\""));
assert!(result.text.contains("filename=\"voice.ogg\""));
assert!(result.text.contains("duration=\"5s\""));
assert!(result.text.contains("Transcript: Hello, can you help me?"));
assert!(result.text.ends_with("</attachments>"));
assert!(result.image_parts.is_empty());
}
#[test]
fn audio_without_transcript() {
let mut att = make_attachment(AttachmentKind::Audio);
att.filename = Some("voice.ogg".to_string());
att.duration_secs = Some(10);
let result = augment_with_attachments("hi", &[att]).unwrap();
assert!(result.text.contains("Audio transcript unavailable."));
assert!(result.text.contains("duration=\"10s\""));
}
#[test]
fn image_without_data_no_visual() {
let mut att = make_attachment(AttachmentKind::Image);
att.filename = Some("screenshot.png".to_string());
att.mime_type = "image/png".to_string();
att.size_bytes = Some(245_000);
let result = augment_with_attachments("check this", &[att]).unwrap();
assert!(result.text.contains("type=\"image\""));
assert!(result.text.contains("filename=\"screenshot.png\""));
assert!(result.text.contains("mime=\"image/png\""));
assert!(result.text.contains("size=\"239KB\""));
assert!(
result
.text
.contains("[Image attached — visual content not available in this conversation]")
);
assert!(result.image_parts.is_empty());
}
#[test]
fn image_with_data_produces_content_part() {
let mut att = make_attachment(AttachmentKind::Image);
att.filename = Some("photo.jpg".to_string());
att.mime_type = "image/jpeg".to_string();
att.data = vec![0xFF, 0xD8, 0xFF]; // fake JPEG header
let result = augment_with_attachments("look", &[att]).unwrap();
assert!(
result
.text
.contains("[Image attached — sent as visual content]")
);
assert_eq!(result.image_parts.len(), 1);
match &result.image_parts[0] {
ContentPart::ImageUrl { image_url } => {
assert!(image_url.url.starts_with("data:image/jpeg;base64,"));
}
other => panic!("Expected ImageUrl, got: {:?}", other),
}
}
#[test]
fn document_with_extracted_text() {
let mut att = make_attachment(AttachmentKind::Document);
att.filename = Some("report.pdf".to_string());
att.extracted_text = Some("Executive summary: Q3 results".to_string());
let result = augment_with_attachments("review", &[att]).unwrap();
assert!(result.text.contains("type=\"document\""));
assert!(result.text.contains("filename=\"report.pdf\""));
assert!(result.text.contains("Executive summary: Q3 results"));
}
#[test]
fn document_without_extracted_text() {
let mut att = make_attachment(AttachmentKind::Document);
att.filename = Some("data.csv".to_string());
att.mime_type = "text/csv".to_string();
att.size_bytes = Some(1024);
let result = augment_with_attachments("analyze", &[att]).unwrap();
assert!(result.text.contains("type=\"document\""));
assert!(result.text.contains("mime=\"text/csv\""));
assert!(
result
.text
.contains("[Document attached — text extraction unavailable]")
);
}
#[test]
fn multiple_attachments_with_mixed_images() {
let mut audio = make_attachment(AttachmentKind::Audio);
audio.filename = Some("voice.ogg".to_string());
audio.extracted_text = Some("Hello".to_string());
let mut image_with_data = make_attachment(AttachmentKind::Image);
image_with_data.filename = Some("photo.jpg".to_string());
image_with_data.mime_type = "image/jpeg".to_string();
image_with_data.data = vec![0xFF, 0xD8];
let mut image_no_data = make_attachment(AttachmentKind::Image);
image_no_data.filename = Some("remote.png".to_string());
image_no_data.mime_type = "image/png".to_string();
let result =
augment_with_attachments("msg", &[audio, image_with_data, image_no_data]).unwrap();
assert!(result.text.contains("index=\"1\""));
assert!(result.text.contains("index=\"2\""));
assert!(result.text.contains("index=\"3\""));
// Only the image with data produces a content part
assert_eq!(result.image_parts.len(), 1);
}
#[test]
fn original_content_preserved() {
let original = "Please help me with this task";
let mut att = make_attachment(AttachmentKind::Audio);
att.extracted_text = Some("transcript".to_string());
let result = augment_with_attachments(original, &[att]).unwrap();
assert!(result.text.starts_with(original));
}
}
+9 -118
View File
@@ -68,10 +68,7 @@ impl Agent {
self.handle_help_job(&message.user_id, &job_id).await?
}
MessageIntent::Command { command, args } => {
match self
.handle_command(&command, &args, &message.channel)
.await?
{
match self.handle_command(&command, &args).await? {
Some(s) => s,
None => return Ok(SubmissionResult::Ok { message: None }), // Shutdown signal
}
@@ -345,6 +342,7 @@ impl Agent {
crate::workspace::hygiene::HygieneConfig::default(),
workspace.clone(),
self.llm().clone(),
self.safety().clone(),
);
match runner.check_heartbeat().await {
@@ -405,7 +403,7 @@ impl Agent {
.with_max_tokens(512)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm().clone());
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
match reasoning.complete(request).await {
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
"Thread Summary:\n\n{}",
@@ -453,7 +451,7 @@ impl Agent {
.with_max_tokens(512)
.with_temperature(0.5);
let reasoning = Reasoning::new(self.llm().clone());
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
match reasoning.complete(request).await {
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
"Suggested Next Steps:\n\n{}",
@@ -468,7 +466,6 @@ impl Agent {
&self,
command: &str,
args: &[String],
channel: &str,
) -> Result<SubmissionResult, Error> {
match command {
"help" => Ok(SubmissionResult::response(concat!(
@@ -504,75 +501,12 @@ impl Agent {
" /heartbeat Run heartbeat check\n",
" /summarize Summarize current thread\n",
" /suggest Suggest next steps\n",
" /restart Gracefully restart the process\n",
"\n",
" /quit Exit",
))),
"ping" => Ok(SubmissionResult::response("pong!")),
"restart" => {
tracing::info!("[commands::restart] Restart command received");
// Channel authorization check: restart is only available via web interface
if channel != "gateway" {
tracing::warn!(
"[commands::restart] Restart rejected: not from gateway channel (from: {})",
channel
);
return Ok(SubmissionResult::error(
"Restart is only available through the web interface with explicit user confirmation. \
Use the Restart button in the UI."
.to_string(),
));
}
// Environment check: restart is only available in Docker containers
let in_docker = std::env::var("IRONCLAW_IN_DOCKER")
.map(|v| v.to_lowercase() == "true")
.unwrap_or(false);
tracing::debug!("[commands::restart] IRONCLAW_IN_DOCKER={}", in_docker);
if !in_docker {
tracing::warn!(
"[commands::restart] Restart rejected: not in Docker environment"
);
return Ok(SubmissionResult::error(
"Restart is not available in this environment. \
The IRONCLAW_IN_DOCKER environment variable must be set to 'true' for Docker deployments."
.to_string(),
));
}
// Execute restart tool directly (don't dispatch as a job for LLM planning)
// This ensures the tool runs immediately without LLM involvement
use crate::tools::Tool;
let tool = crate::tools::builtin::RestartTool;
let params = serde_json::json!({});
// Create a minimal JobContext for the tool
let dummy_ctx =
crate::context::JobContext::with_user("system", "Restart", "Graceful restart");
match tool.execute(params, &dummy_ctx).await {
Ok(output) => {
tracing::info!("[commands::restart] RestartTool executed successfully");
// Extract text from the ToolOutput result
let response = match output.result {
serde_json::Value::String(s) => s,
_ => output.result.to_string(),
};
Ok(SubmissionResult::response(response))
}
Err(e) => {
tracing::error!(
"[commands::restart] RestartTool execution failed: {:?}",
e
);
Ok(SubmissionResult::error(format!("Restart failed: {}", e)))
}
}
}
"version" => Ok(SubmissionResult::response(format!(
"{} v{}",
env!("CARGO_PKG_NAME"),
@@ -662,14 +596,10 @@ impl Agent {
}
match self.llm().set_model(requested) {
Ok(()) => {
// Persist the model choice so it survives restarts.
self.persist_selected_model(requested).await;
Ok(SubmissionResult::response(format!(
"Switched model to: {}",
requested
)))
}
Ok(()) => Ok(SubmissionResult::response(format!(
"Switched model to: {}",
requested
))),
Err(e) => Ok(SubmissionResult::error(format!(
"Failed to switch model: {}",
e
@@ -814,53 +744,14 @@ impl Agent {
&self,
command: &str,
args: &[String],
channel: &str,
) -> Result<Option<String>, Error> {
// System commands are now handled directly via Submission::SystemCommand,
// but the router may still send us unknown /commands.
match self.handle_system_command(command, args, channel).await? {
match self.handle_system_command(command, args).await? {
SubmissionResult::Response { content } => Ok(Some(content)),
SubmissionResult::Ok { message } => Ok(message),
SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))),
_ => Ok(None),
}
}
/// Persist the selected model to the settings store (DB and/or TOML config).
///
/// Best-effort: logs warnings on failure but does not propagate errors,
/// since the in-memory model switch already succeeded.
async fn persist_selected_model(&self, model: &str) {
// 1. Persist to DB if available.
if let Some(store) = self.store() {
let value = serde_json::Value::String(model.to_string());
if let Err(e) = store.set_setting("default", "selected_model", &value).await {
tracing::warn!("Failed to persist model to DB: {}", e);
}
}
// 2. Update TOML config file if it exists (sync I/O in spawn_blocking).
let model_owned = model.to_string();
if let Err(e) = tokio::task::spawn_blocking(move || {
let toml_path = crate::settings::Settings::default_toml_path();
match crate::settings::Settings::load_toml(&toml_path) {
Ok(Some(mut settings)) => {
settings.selected_model = Some(model_owned);
if let Err(e) = settings.save_toml(&toml_path) {
tracing::warn!("Failed to persist model to config.toml: {}", e);
}
}
Ok(None) => {
// No config file on disk; nothing to update.
}
Err(e) => {
tracing::warn!("Failed to load config.toml for model persistence: {}", e);
}
}
})
.await
{
tracing::warn!("Model TOML persistence task failed: {}", e);
}
}
}
+37 -112
View File
@@ -13,6 +13,7 @@ use crate::agent::context_monitor::{CompactionStrategy, ContextBreakdown};
use crate::agent::session::Thread;
use crate::error::Error;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
use crate::safety::SafetyLayer;
use crate::workspace::Workspace;
/// Result of a compaction operation.
@@ -33,12 +34,13 @@ pub struct CompactionResult {
/// Compacts conversation context to stay within limits.
pub struct ContextCompactor {
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
}
impl ContextCompactor {
/// Create a new context compactor.
pub fn new(llm: Arc<dyn LlmProvider>) -> Self {
Self { llm }
pub fn new(llm: Arc<dyn LlmProvider>, safety: Arc<SafetyLayer>) -> Self {
Self { llm, safety }
}
/// Compact a thread's context using the given strategy.
@@ -103,26 +105,27 @@ impl ContextCompactor {
// Generate summary
let summary = self.generate_summary(&to_summarize).await?;
// Write to workspace if available.
// If archival fails, preserve turns to avoid context loss.
let (summary_written, turns_removed) = if let Some(ws) = workspace {
// Write to workspace if available
let summary_written = if let Some(ws) = workspace {
match self.write_summary_to_workspace(ws, &summary).await {
Ok(()) => {
thread.truncate_turns(keep_recent);
(true, turns_to_remove)
}
Ok(()) => true,
Err(e) => {
tracing::warn!("Compaction summary write failed (turns preserved): {}", e);
(false, 0)
tracing::warn!(
"Compaction summary write failed (turns will still be truncated): {}",
e
);
false
}
}
} else {
thread.truncate_turns(keep_recent);
(false, turns_to_remove)
false
};
// Truncate thread
thread.truncate_turns(keep_recent);
Ok(CompactionPartial {
turns_removed,
turns_removed: turns_to_remove,
summary_written,
summary: Some(summary),
})
@@ -164,20 +167,23 @@ impl ContextCompactor {
// Format turns for storage
let content = format_turns_for_storage(old_turns);
// Write to workspace. If archival fails, preserve turns.
let (written, turns_removed) = match self.write_context_to_workspace(ws, &content).await {
Ok(()) => {
thread.truncate_turns(keep_recent);
(true, turns_to_remove)
}
// Write to workspace
let written = match self.write_context_to_workspace(ws, &content).await {
Ok(()) => true,
Err(e) => {
tracing::warn!("Compaction context write failed (turns preserved): {}", e);
(false, 0)
tracing::warn!(
"Compaction context write failed (turns will still be truncated): {}",
e
);
false
}
};
// Truncate
thread.truncate_turns(keep_recent);
Ok(CompactionPartial {
turns_removed,
turns_removed: turns_to_remove,
summary_written: written,
summary: None,
})
@@ -227,7 +233,7 @@ Be brief but capture all important details. Use bullet points."#,
.with_max_tokens(1024)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm.clone());
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
let (text, _) = reasoning.complete(request).await?;
Ok(text)
}
@@ -340,11 +346,17 @@ mod tests {
// === QA Plan - Compaction strategy tests ===
use crate::agent::context_monitor::CompactionStrategy;
use crate::config::SafetyConfig;
use crate::safety::SafetyLayer;
use crate::testing::StubLlm;
/// Helper: build a `ContextCompactor` with the given `StubLlm`.
fn make_compactor(llm: Arc<StubLlm>) -> ContextCompactor {
ContextCompactor::new(llm)
let safety = Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
}));
ContextCompactor::new(llm, safety)
}
/// Helper: build a thread with `n` completed turns.
@@ -358,19 +370,6 @@ mod tests {
thread
}
#[cfg(feature = "libsql")]
async fn make_unmigrated_workspace() -> crate::workspace::Workspace {
use crate::db::Database;
use crate::db::libsql::LibSqlBackend;
// Intentionally skip migrations so workspace append operations fail.
let backend = LibSqlBackend::new_memory()
.await
.expect("should create in-memory libsql backend");
let db: Arc<dyn Database> = Arc::new(backend);
crate::workspace::Workspace::new_with_db("compaction-test", db)
}
// ------------------------------------------------------------------
// 1. compact_truncate keeps last N turns
// ------------------------------------------------------------------
@@ -569,43 +568,6 @@ mod tests {
assert_eq!(llm.calls(), 0);
}
#[cfg(feature = "libsql")]
#[tokio::test]
async fn test_compact_with_summary_preserves_turns_when_workspace_write_fails() {
let llm = Arc::new(StubLlm::new("summary"));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(8);
let original_inputs: Vec<String> =
thread.turns.iter().map(|t| t.user_input.clone()).collect();
let workspace = make_unmigrated_workspace().await;
let result = compactor
.compact(
&mut thread,
CompactionStrategy::Summarize { keep_recent: 3 },
Some(&workspace),
)
.await
.expect("compact should succeed even when workspace write fails");
// On archival failure, no turns should be removed.
assert_eq!(thread.turns.len(), 8);
assert_eq!(
thread
.turns
.iter()
.map(|t| t.user_input.as_str())
.collect::<Vec<_>>(),
original_inputs
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
);
assert_eq!(result.turns_removed, 0);
assert!(!result.summary_written);
assert_eq!(llm.calls(), 1);
}
// ------------------------------------------------------------------
// 7. compact_to_workspace without workspace falls back to truncation
// ------------------------------------------------------------------
@@ -654,43 +616,6 @@ mod tests {
assert_eq!(result.turns_removed, 0);
}
#[cfg(feature = "libsql")]
#[tokio::test]
async fn test_compact_to_workspace_preserves_turns_when_workspace_write_fails() {
let llm = Arc::new(StubLlm::new("unused"));
let compactor = make_compactor(llm.clone());
let mut thread = make_thread(20);
let original_inputs: Vec<String> =
thread.turns.iter().map(|t| t.user_input.clone()).collect();
let workspace = make_unmigrated_workspace().await;
let result = compactor
.compact(
&mut thread,
CompactionStrategy::MoveToWorkspace,
Some(&workspace),
)
.await
.expect("compact should succeed even when workspace write fails");
// On archival failure, no turns should be removed.
assert_eq!(thread.turns.len(), 20);
assert_eq!(
thread
.turns
.iter()
.map(|t| t.user_input.as_str())
.collect::<Vec<_>>(),
original_inputs
.iter()
.map(|s| s.as_str())
.collect::<Vec<_>>()
);
assert_eq!(result.turns_removed, 0);
assert!(!result.summary_written);
assert_eq!(llm.calls(), 0);
}
// ------------------------------------------------------------------
// 9. format_turns_for_storage includes tool calls
// ------------------------------------------------------------------
+18 -272
View File
@@ -131,12 +131,10 @@ impl CostGuard {
// Check hourly rate
if let Some(limit) = self.config.max_actions_per_hour {
let mut window = self.action_window.lock().await;
// checked_sub avoids panic when system uptime < 1 hour (Windows)
if let Some(cutoff) = Instant::now().checked_sub(std::time::Duration::from_secs(3600)) {
// Drain expired entries
while window.front().is_some_and(|t| *t < cutoff) {
window.pop_front();
}
let cutoff = Instant::now() - std::time::Duration::from_secs(3600);
// Drain expired entries
while window.front().is_some_and(|t| *t < cutoff) {
window.pop_front();
}
let count = window.len() as u64;
if count >= limit {
@@ -153,46 +151,21 @@ impl CostGuard {
/// Record a completed LLM action: its token costs and the action timestamp.
///
/// Call this AFTER an LLM call completes so that costs are tracked.
/// - `cache_read_input_tokens`: tokens served from cache.
/// - `cache_creation_input_tokens`: tokens written to cache.
/// - `cache_read_discount`: divisor for cache-read cost (e.g. 10 for Anthropic 90% off, 2 for OpenAI 50% off).
/// - `cache_write_multiplier`: cost multiplier for cache writes (1.25 for 5m, 2.0 for 1h).
///
/// When `cost_per_token` is `Some`, those rates are used directly (provider-
/// sourced pricing). When `None`, falls back to the static `costs::model_cost`
/// lookup table, then `costs::default_cost`.
#[allow(clippy::too_many_arguments)]
pub async fn record_llm_call(
&self,
model: &str,
input_tokens: u32,
output_tokens: u32,
cache_read_input_tokens: u32,
cache_creation_input_tokens: u32,
cache_read_discount: Decimal,
cache_write_multiplier: Decimal,
cost_per_token: Option<(Decimal, Decimal)>,
) -> Decimal {
let (input_rate, output_rate) = cost_per_token
.unwrap_or_else(|| costs::model_cost(model).unwrap_or_else(costs::default_cost));
// Cached read tokens cost input_rate / cache_read_discount (provider-specific).
// Cached write tokens cost write_multiplier × input_rate (e.g. 1.25× for 5m, 2× for 1h).
// Uncached tokens = total input - cache reads - cache writes.
let cached_total = cache_read_input_tokens.saturating_add(cache_creation_input_tokens);
let uncached_input = input_tokens.saturating_sub(cached_total);
let effective_discount = if cache_read_discount.is_zero() {
Decimal::ONE
} else {
cache_read_discount
};
let cache_read_cost =
input_rate * Decimal::from(cache_read_input_tokens) / effective_discount;
let cache_write_cost =
input_rate * Decimal::from(cache_creation_input_tokens) * cache_write_multiplier;
let cost = input_rate * Decimal::from(uncached_input)
+ cache_read_cost
+ cache_write_cost
+ output_rate * Decimal::from(output_tokens);
let cost =
input_rate * Decimal::from(input_tokens) + output_rate * Decimal::from(output_tokens);
// Update daily cost (reset if new day)
{
@@ -262,11 +235,9 @@ impl CostGuard {
/// Number of actions in the current hourly window.
pub async fn actions_this_hour(&self) -> u64 {
let mut window = self.action_window.lock().await;
// checked_sub avoids panic when system uptime < 1 hour (Windows)
if let Some(cutoff) = Instant::now().checked_sub(std::time::Duration::from_secs(3600)) {
while window.front().is_some_and(|t| *t < cutoff) {
window.pop_front();
}
let cutoff = Instant::now() - std::time::Duration::from_secs(3600);
while window.front().is_some_and(|t| *t < cutoff) {
window.pop_front();
}
window.len() as u64
}
@@ -296,16 +267,7 @@ mod tests {
// Record a big call, still allowed
guard
.record_llm_call(
"gpt-4o",
100_000,
100_000,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.record_llm_call("gpt-4o", 100_000, 100_000, None)
.await;
assert!(guard.check_allowed().await.is_ok());
}
@@ -323,18 +285,7 @@ mod tests {
// Record a call that costs more than $0.01
// gpt-4o: input=$0.0000025/tok, output=$0.00001/tok
// 10000 input + 10000 output = $0.025 + $0.10 = $0.125
guard
.record_llm_call(
"gpt-4o",
10_000,
10_000,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.await;
guard.record_llm_call("gpt-4o", 10_000, 10_000, None).await;
// Now should be blocked
let result = guard.check_allowed().await;
@@ -357,9 +308,7 @@ mod tests {
// First 3 actions allowed
for _ in 0..3 {
assert!(guard.check_allowed().await.is_ok());
guard
.record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
guard.record_llm_call("gpt-4o", 10, 10, None).await;
}
// 4th should be blocked
@@ -380,9 +329,7 @@ mod tests {
assert_eq!(guard.daily_spend().await, Decimal::ZERO);
let cost = guard
.record_llm_call("gpt-4o", 1000, 500, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
let cost = guard.record_llm_call("gpt-4o", 1000, 500, None).await;
assert!(cost > Decimal::ZERO);
assert_eq!(guard.daily_spend().await, cost);
}
@@ -393,12 +340,8 @@ mod tests {
assert_eq!(guard.actions_this_hour().await, 0);
guard
.record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
guard
.record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
guard.record_llm_call("gpt-4o", 10, 10, None).await;
guard.record_llm_call("gpt-4o", 10, 10, None).await;
assert_eq!(guard.actions_this_hour().await, 2);
}
@@ -435,23 +378,10 @@ mod tests {
assert!(guard.model_usage().await.is_empty());
// Record calls for two different models
guard.record_llm_call("gpt-4o", 1000, 500, None).await;
guard.record_llm_call("gpt-4o", 2000, 1000, None).await;
guard
.record_llm_call("gpt-4o", 1000, 500, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
guard
.record_llm_call("gpt-4o", 2000, 1000, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
guard
.record_llm_call(
"claude-3-5-sonnet-20241022",
500,
200,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.record_llm_call("claude-3-5-sonnet-20241022", 500, 200, None)
.await;
let usage = guard.model_usage().await;
@@ -472,188 +402,4 @@ mod tests {
// Costs should differ since models have different pricing
assert_ne!(gpt.cost, claude.cost);
}
#[tokio::test]
async fn test_cache_discount_reduces_cost() {
let guard = CostGuard::new(CostGuardConfig::default());
// Full price: 1000 input + 500 output, no cache
let full_cost = guard
.record_llm_call(
"claude-opus-4-6",
1000,
500,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.await;
let guard2 = CostGuard::new(CostGuardConfig::default());
// Same tokens but all input cached (90% discount on input)
let cached_cost = guard2
.record_llm_call(
"claude-opus-4-6",
1000,
500,
1000,
0,
dec!(10),
Decimal::ONE,
None,
)
.await;
// Cached cost must be strictly less than full cost
assert!(
cached_cost < full_cost,
"cached_cost ({}) should be less than full_cost ({})",
cached_cost,
full_cost
);
// The difference should be exactly 90% of the input cost
let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap();
let expected_savings = input_rate * Decimal::from(1000u32) * dec!(9) / dec!(10);
let actual_savings = full_cost - cached_cost;
assert_eq!(
actual_savings, expected_savings,
"savings should be 90% of input cost for fully-cached request"
);
}
#[tokio::test]
async fn test_cache_write_surcharge_increases_cost() {
let guard = CostGuard::new(CostGuardConfig::default());
// Full price: 1000 input + 500 output, no cache activity
let full_cost = guard
.record_llm_call(
"claude-opus-4-6",
1000,
500,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.await;
let guard2 = CostGuard::new(CostGuardConfig::default());
// Same tokens, but all input tokens are cache writes (1.25x surcharge for 5m TTL)
let short_multiplier = Decimal::new(125, 2); // 1.25
let write_cost = guard2
.record_llm_call(
"claude-opus-4-6",
1000,
500,
0,
1000,
Decimal::ONE,
short_multiplier,
None,
)
.await;
// Write cost must be strictly greater than full cost
assert!(
write_cost > full_cost,
"write_cost ({}) should be greater than full_cost ({})",
write_cost,
full_cost
);
// The difference should be exactly 25% of the input cost
let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap();
let expected_surcharge = input_rate * Decimal::from(1000u32) * dec!(0.25);
let actual_surcharge = write_cost - full_cost;
assert_eq!(
actual_surcharge, expected_surcharge,
"surcharge should be 25% of input cost for 5m cache writes"
);
}
#[tokio::test]
async fn test_cache_write_surcharge_long_ttl() {
let guard = CostGuard::new(CostGuardConfig::default());
// Full price: 1000 input + 500 output
let full_cost = guard
.record_llm_call(
"claude-opus-4-6",
1000,
500,
0,
0,
Decimal::ONE,
Decimal::ONE,
None,
)
.await;
let guard2 = CostGuard::new(CostGuardConfig::default());
// All input tokens are cache writes with 2.0x multiplier (1h TTL)
let long_multiplier = Decimal::TWO;
let write_cost = guard2
.record_llm_call(
"claude-opus-4-6",
1000,
500,
0,
1000,
Decimal::ONE,
long_multiplier,
None,
)
.await;
// Write cost > full cost
assert!(write_cost > full_cost);
// Surcharge should be 100% of input cost (2.0x - 1.0x = 1.0x)
let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap();
let expected_surcharge = input_rate * Decimal::from(1000u32);
let actual_surcharge = write_cost - full_cost;
assert_eq!(
actual_surcharge, expected_surcharge,
"surcharge should be 100% of input cost for 1h cache writes"
);
}
/// Regression test for #657: Instant::now() - Duration panics on Windows
/// when system uptime is less than the subtracted duration.
#[tokio::test]
async fn test_checked_sub_no_panic_on_fresh_guard() {
// A fresh CostGuard with rate limits should not panic even if
// checked_sub returns None (simulating short uptime).
let guard = CostGuard::new(CostGuardConfig {
max_cost_per_day_cents: None,
max_actions_per_hour: Some(100),
});
// These must not panic regardless of system uptime
assert!(guard.check_allowed().await.is_ok());
assert_eq!(guard.actions_this_hour().await, 0);
// Record some actions and verify again
guard
.record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None)
.await;
assert!(guard.check_allowed().await.is_ok());
assert_eq!(guard.actions_this_hour().await, 1);
}
/// Verify that checked_sub itself behaves as expected for the pattern we use.
#[test]
fn test_instant_checked_sub_returns_none_for_overflow() {
// Duration::MAX will always exceed uptime, so checked_sub must return None
let result = Instant::now().checked_sub(std::time::Duration::MAX);
assert!(result.is_none());
}
}
+53 -367
View File
@@ -15,7 +15,6 @@ use crate::channels::{IncomingMessage, StatusUpdate};
use crate::context::JobContext;
use crate::error::Error;
use crate::llm::{ChatMessage, Reasoning, ReasoningContext, RespondResult};
use crate::tools::redact_params;
/// Result of the agentic loop execution.
pub(super) enum AgenticLoopResult {
@@ -50,18 +49,8 @@ impl Agent {
// Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.)
// In group chats, MEMORY.md is excluded to prevent leaking personal context.
// Resolve the user's timezone
let user_tz = crate::timezone::resolve_timezone(
message.timezone.as_deref(),
None, // user setting lookup can be added later
&self.config.default_timezone,
);
let system_prompt = if let Some(ws) = self.workspace() {
match ws
.system_prompt_for_context_tz(is_group_chat, user_tz)
.await
{
match ws.system_prompt_for_context(is_group_chat).await {
Ok(prompt) if !prompt.is_empty() => Some(prompt),
Ok(_) => None,
Err(e) => {
@@ -113,7 +102,7 @@ impl Agent {
None
};
let mut reasoning = Reasoning::new(self.llm().clone())
let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone())
.with_channel(message.channel.clone())
.with_model_name(self.llm().active_model_name())
.with_group_chat(is_group_chat);
@@ -137,21 +126,7 @@ impl Agent {
let mut context_messages = initial_messages;
// Create a JobContext for tool execution (chat doesn't have a real job)
let mut job_ctx =
JobContext::with_user(&message.user_id, "chat", "Interactive chat session");
job_ctx.http_interceptor = self.deps.http_interceptor.clone();
job_ctx.user_timezone = user_tz.name().to_string();
// Build system prompts once for this turn. Two variants: with tools
// (normal iterations) and without (force_text final iteration).
let initial_tool_defs = self.tools().tool_definitions().await;
let initial_tool_defs = if !active_skills.is_empty() {
crate::skills::attenuate_tools(&initial_tool_defs, &active_skills).tools
} else {
initial_tool_defs
};
let cached_prompt = reasoning.build_system_prompt_with_tools(&initial_tool_defs);
let cached_prompt_no_tools = reasoning.build_system_prompt_with_tools(&[]);
let job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session");
let max_tool_iterations = self.config.max_tool_iterations;
// Force a text-only response on the last iteration to guarantee termination
@@ -160,8 +135,6 @@ impl Agent {
let force_text_at = max_tool_iterations;
let nudge_at = max_tool_iterations.saturating_sub(1);
let mut iteration = 0;
const MAX_TOOL_INTENT_NUDGES: u32 = 2;
let mut consecutive_tool_intent_nudges: u32 = 0;
loop {
iteration += 1;
// Hard ceiling one past the forced-text iteration (should never be reached
@@ -230,16 +203,10 @@ impl Agent {
};
// Call LLM with current context; force_text drops tools to guarantee a
// text response on the final iteration. The pre-built system prompt
// avoids rebuilding the same ~1,500-token string each iteration.
// text response on the final iteration.
let mut context = ReasoningContext::new()
.with_messages(context_messages.clone())
.with_tools(tool_defs)
.with_system_prompt(if force_text {
cached_prompt_no_tools.clone()
} else {
cached_prompt.clone()
})
.with_metadata({
let mut m = std::collections::HashMap::new();
m.insert("thread_id".to_string(), thread_id.to_string());
@@ -276,7 +243,7 @@ impl Agent {
// Compact: keep system messages + last user message + current turn
context_messages = compact_messages_for_retry(&context_messages);
// Rebuild context with compacted messages, reusing cached prompt
// Rebuild context with compacted messages
let mut retry_context = ReasoningContext::new()
.with_messages(context_messages.clone())
.with_tools(if force_text {
@@ -286,7 +253,6 @@ impl Agent {
})
.with_metadata(context.metadata.clone());
retry_context.force_text = force_text;
retry_context.system_prompt = context.system_prompt.clone();
reasoning
.respond_with_tools(&retry_context)
@@ -307,18 +273,12 @@ impl Agent {
// Record cost and track token usage
let model_name = self.llm().active_model_name();
let read_discount = self.llm().cache_read_discount();
let write_multiplier = self.llm().cache_write_multiplier();
let call_cost = self
.cost_guard()
.record_llm_call(
&model_name,
output.usage.input_tokens,
output.usage.output_tokens,
output.usage.cache_read_input_tokens,
output.usage.cache_creation_input_tokens,
read_discount,
write_multiplier,
Some(self.llm().cost_per_token()),
)
.await;
@@ -331,35 +291,12 @@ impl Agent {
match output.result {
RespondResult::Text(text) => {
// Nudge the LLM if it expressed tool intent without calling tools.
// This is common with non-Anthropic models (e.g. GLM-5 via NEAR AI)
// that output "Let me search…" but don't issue tool_calls.
if !force_text
&& !context.available_tools.is_empty()
&& consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES
&& crate::llm::llm_signals_tool_intent(&text)
{
consecutive_tool_intent_nudges += 1;
tracing::info!(
iteration,
"LLM expressed tool intent without calling a tool, nudging"
);
context_messages.push(ChatMessage::assistant(&text));
context_messages.push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE));
continue;
}
// Strip internal "[Called tool ...]" text that can leak when
// provider flattening (e.g. NEAR AI) converts tool_calls to
// plain text and the LLM echoes it back.
let sanitized = strip_internal_tool_call_text(&text);
return Ok(AgenticLoopResult::Response(sanitized));
return Ok(AgenticLoopResult::Response(text));
}
RespondResult::ToolCalls {
tool_calls,
content,
} => {
consecutive_tool_intent_nudges = 0;
// Add the assistant message with tool_calls to context.
// OpenAI protocol requires this before tool-result messages.
context_messages.push(ChatMessage::assistant_with_tool_calls(
@@ -380,25 +317,14 @@ impl Agent {
)
.await;
// Record tool calls in the thread with sensitive params redacted.
// Look up each tool's sensitive_params before acquiring the session lock.
// Record tool calls in the thread
{
let mut redacted_args: Vec<serde_json::Value> =
Vec::with_capacity(tool_calls.len());
for tc in &tool_calls {
let safe = if let Some(tool) = self.tools().get(&tc.name).await {
redact_params(&tc.arguments, tool.sensitive_params())
} else {
tc.arguments.clone()
};
redacted_args.push(safe);
}
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id)
&& let Some(turn) = thread.last_turn_mut()
{
for (tc, safe_args) in tool_calls.iter().zip(redacted_args) {
turn.record_tool_call(&tc.name, safe_args);
for tc in &tool_calls {
turn.record_tool_call(&tc.name, tc.arguments.clone());
}
}
}
@@ -427,22 +353,11 @@ impl Agent {
for (idx, original_tc) in tool_calls.iter().enumerate() {
let mut tc = original_tc.clone();
// Fetch the tool upfront so we can redact sensitive params
// before they touch hooks or approval display.
let tool_opt = self.tools().get(&tc.name).await;
let sensitive = tool_opt
.as_ref()
.map(|t| t.sensitive_params())
.unwrap_or(&[]);
// Hook: BeforeToolCall (runs before approval so hooks can
// modify parameters — approval is checked on final params).
// Hooks receive redacted params so sensitive values are not
// exposed to hook handlers or their logs.
let hook_params = redact_params(&tc.arguments, sensitive);
// modify parameters — approval is checked on final params)
let event = crate::hooks::HookEvent::ToolCall {
tool_name: tc.name.clone(),
parameters: hook_params,
parameters: tc.arguments.clone(),
user_id: message.user_id.clone(),
context: "chat".to_string(),
};
@@ -469,20 +384,8 @@ impl Agent {
}
Ok(crate::hooks::HookOutcome::Continue {
modified: Some(new_params),
}) => match serde_json::from_str::<serde_json::Value>(&new_params) {
Ok(mut parsed) => {
// Restore original sensitive param values so a hook
// cannot overwrite them (they were sent as [REDACTED]).
if let Some(obj) = parsed.as_object_mut() {
for key in sensitive {
if let Some(orig_val) = original_tc.arguments.get(*key)
{
obj.insert((*key).to_string(), orig_val.clone());
}
}
}
tc.arguments = parsed;
}
}) => match serde_json::from_str(&new_params) {
Ok(parsed) => tc.arguments = parsed,
Err(e) => {
tracing::warn!(
tool = %tc.name,
@@ -497,7 +400,7 @@ impl Agent {
// Check if tool requires approval on the final (post-hook)
// parameters. Skipped when auto_approve_tools is set.
if !self.config.auto_approve_tools
&& let Some(tool) = tool_opt
&& let Some(tool) = self.tools().get(&tc.name).await
{
use crate::tools::ApprovalRequirement;
let needs_approval = match tool.requires_approval(&tc.arguments) {
@@ -544,17 +447,14 @@ impl Agent {
.execute_chat_tool(&tc.name, &tc.arguments, &job_ctx)
.await;
let disp_tool = self.tools().get(&tc.name).await;
let _ = self
.channels
.send_status(
&message.channel,
StatusUpdate::tool_completed(
tc.name.clone(),
&result,
&tc.arguments,
disp_tool.as_deref(),
),
StatusUpdate::ToolCompleted {
name: tc.name.clone(),
success: result.is_ok(),
},
&message.metadata,
)
.await;
@@ -595,16 +495,13 @@ impl Agent {
)
.await;
let par_tool = tools.get(&tc.name).await;
let _ = channels
.send_status(
&channel,
StatusUpdate::tool_completed(
tc.name.clone(),
&result,
&tc.arguments,
par_tool.as_deref(),
),
StatusUpdate::ToolCompleted {
name: tc.name.clone(),
success: result.is_ok(),
},
&metadata,
)
.await;
@@ -681,53 +578,8 @@ impl Agent {
.into())
});
// Detect image generation sentinel in tool output
// (only from image tools — avoids parsing all tool outputs)
let is_image_sentinel = if let Ok(ref output) = tool_result
&& matches!(tc.name.as_str(), "image_generate" | "image_edit")
{
if let Ok(sentinel) =
serde_json::from_str::<serde_json::Value>(output)
&& sentinel.get("type").and_then(|v| v.as_str())
== Some("image_generated")
{
let data_url = sentinel
.get("data")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
let path = sentinel
.get("path")
.and_then(|v| v.as_str())
.map(String::from);
// Skip broadcasting if data_url is empty to avoid
// sending a broken ImageGenerated SSE event.
if data_url.is_empty() {
tracing::warn!(
"Image generation sentinel has empty data URL, skipping broadcast"
);
} else {
let _ = self
.channels
.send_status(
&message.channel,
StatusUpdate::ImageGenerated { data_url, path },
&message.metadata,
)
.await;
}
true
} else {
false
}
} else {
false
};
// Send ToolResult preview (skip for image sentinels to avoid
// broadcasting multi-MB base64 data as a preview)
if !is_image_sentinel
&& let Ok(ref output) = tool_result
// Send ToolResult preview
if let Ok(ref output) = tool_result
&& !output.is_empty()
{
let _ = self
@@ -743,6 +595,23 @@ impl Agent {
.await;
}
// Record result in thread
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id)
&& let Some(turn) = thread.last_turn_mut()
{
match &tool_result {
Ok(output) => {
turn.record_tool_result(serde_json::json!(output));
}
Err(e) => {
turn.record_tool_error(e.to_string());
}
}
}
}
// Check for auth awaiting — defer the return
// until all results are recorded.
if deferred_auth.is_none()
@@ -772,17 +641,7 @@ impl Agent {
deferred_auth = Some(instructions);
}
// Stash full output so subsequent tools can reference it
if let Ok(ref output) = tool_result {
job_ctx
.tool_output_stash
.write()
.await
.insert(tc.id.clone(), output.clone());
}
// Sanitize and add tool result to context
let is_tool_error = tool_result.is_err();
let result_content = match tool_result {
Ok(output) => {
let sanitized =
@@ -793,26 +652,9 @@ impl Agent {
sanitized.was_modified,
)
}
Err(e) => format!("Tool '{}' failed: {}", tc.name, e),
Err(e) => format!("Error: {}", e),
};
// Record sanitized result in thread so messages()
// and persist_tool_calls() use cleaned content.
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id)
&& let Some(turn) = thread.last_turn_mut()
{
if is_tool_error {
turn.record_tool_error(result_content.clone());
} else {
turn.record_tool_result(serde_json::json!(
result_content
));
}
}
}
context_messages.push(ChatMessage::tool_result(
&tc.id,
&tc.name,
@@ -829,20 +671,14 @@ impl Agent {
// Handle approval if a tool needed it
if let Some((approval_idx, tc, tool)) = approval_needed {
// Show redacted params in the approval UI — the user already knows
// the sensitive value (they provided it); showing it again is
// unnecessary and creates a leakage path through channel logs.
let display_params = redact_params(&tc.arguments, tool.sensitive_params());
let pending = PendingApproval {
request_id: Uuid::new_v4(),
tool_name: tc.name.clone(),
parameters: tc.arguments.clone(),
display_parameters: display_params,
description: tool.description().to_string(),
tool_call_id: tc.id.clone(),
context_messages: context_messages.clone(),
deferred_tool_calls: tool_calls[approval_idx + 1..].to_vec(),
user_timezone: Some(user_tz.name().to_string()),
};
return Ok(AgenticLoopResult::NeedApproval { pending });
@@ -898,10 +734,9 @@ pub(super) async fn execute_chat_tool_standalone(
.into());
}
let safe_params = redact_params(params, tool.sensitive_params());
tracing::debug!(
tool = %tool_name,
params = %safe_params,
params = %params,
"Tool call started"
);
@@ -1065,38 +900,6 @@ fn compact_messages_for_retry(messages: &[ChatMessage]) -> Vec<ChatMessage> {
compacted
}
/// Strip internal `[Called tool ...]` and `[Tool ... returned: ...]` markers
/// from a response string. These markers are inserted by provider-level message
/// flattening (e.g. NEAR AI) and can leak into the user-visible response when
/// the LLM echoes them back.
fn strip_internal_tool_call_text(text: &str) -> String {
// Remove lines that are purely internal tool-call markers.
// Pattern: lines matching `[Called tool <name>(...)]` or `[Tool <name> returned: ...]`
let result = text
.lines()
.filter(|line| {
let trimmed = line.trim();
!((trimmed.starts_with("[Called tool ") && trimmed.ends_with(']'))
|| (trimmed.starts_with("[Tool ")
&& trimmed.contains(" returned:")
&& trimmed.ends_with(']')))
})
.fold(String::new(), |mut acc, s| {
if !acc.is_empty() {
acc.push('\n');
}
acc.push_str(s);
acc
});
let result = result.trim();
if result.is_empty() {
"I wasn't able to complete that request. Could you try rephrasing or providing more details?".to_string()
} else {
result.to_string()
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
@@ -1144,8 +947,6 @@ mod tests {
input_tokens: 0,
output_tokens: 0,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
@@ -1159,8 +960,6 @@ mod tests {
input_tokens: 0,
output_tokens: 0,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
}
@@ -1183,10 +982,6 @@ mod tests {
skills_config: SkillsConfig::default(),
hooks: Arc::new(HookRegistry::new()),
cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())),
sse_tx: None,
http_interceptor: None,
transcription: None,
document_extraction: None,
};
Agent::new(
@@ -1204,7 +999,6 @@ mod tests {
max_actions_per_hour: None,
max_tool_iterations: 50,
auto_approve_tools: false,
default_timezone: "UTC".to_string(),
},
deps,
Arc::new(ChannelManager::new()),
@@ -1291,7 +1085,6 @@ mod tests {
request_id: uuid::Uuid::new_v4(),
tool_name: "shell".to_string(),
parameters: serde_json::json!({"command": "echo hi"}),
display_parameters: serde_json::json!({"command": "echo hi"}),
description: "Run shell command".to_string(),
tool_call_id: "call_1".to_string(),
context_messages: vec![],
@@ -1307,7 +1100,6 @@ mod tests {
arguments: serde_json::json!({"message": "done"}),
},
],
user_timezone: None,
};
let json = serde_json::to_string(&pending).expect("serialize");
@@ -1655,8 +1447,12 @@ mod tests {
use crate::testing::StubLlm;
let stub = Arc::new(StubLlm::failing_non_transient("ctx-bomb"));
let safety = Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
}));
let reasoning = Reasoning::new(stub.clone());
let reasoning = Reasoning::new(stub.clone(), safety);
// Build a fat context with lots of history.
let messages = vec![
@@ -1721,8 +1517,6 @@ mod tests {
input_tokens: 0,
output_tokens: 5,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
@@ -1738,8 +1532,6 @@ mod tests {
input_tokens: 0,
output_tokens: 5,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
});
}
// Tools available: always call one.
@@ -1753,8 +1545,6 @@ mod tests {
input_tokens: 0,
output_tokens: 5,
finish_reason: FinishReason::ToolUse,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
}
@@ -1766,7 +1556,11 @@ mod tests {
use crate::llm::{Reasoning, ReasoningContext, RespondResult, ToolDefinition};
let provider = Arc::new(AlwaysToolCallProvider);
let reasoning = Reasoning::new(provider);
let safety = Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
}));
let reasoning = Reasoning::new(provider, safety);
let tool_def = ToolDefinition {
name: "echo".to_string(),
@@ -1875,8 +1669,6 @@ mod tests {
input_tokens: 0,
output_tokens: 2,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
@@ -1891,8 +1683,6 @@ mod tests {
input_tokens: 0,
output_tokens: 2,
finish_reason: FinishReason::Stop,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
});
}
// Always call a tool that does not exist in the registry.
@@ -1906,8 +1696,6 @@ mod tests {
input_tokens: 0,
output_tokens: 5,
finish_reason: FinishReason::ToolUse,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
})
}
}
@@ -1931,10 +1719,6 @@ mod tests {
skills_config: SkillsConfig::default(),
hooks: Arc::new(HookRegistry::new()),
cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())),
sse_tx: None,
http_interceptor: None,
transcription: None,
document_extraction: None,
};
Agent::new(
@@ -1952,7 +1736,6 @@ mod tests {
max_actions_per_hour: None,
max_tool_iterations,
auto_approve_tools: true,
default_timezone: "UTC".to_string(),
},
deps,
Arc::new(ChannelManager::new()),
@@ -2047,10 +1830,6 @@ mod tests {
skills_config: SkillsConfig::default(),
hooks: Arc::new(HookRegistry::new()),
cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())),
sse_tx: None,
http_interceptor: None,
transcription: None,
document_extraction: None,
};
Agent::new(
@@ -2068,7 +1847,6 @@ mod tests {
max_actions_per_hour: None,
max_tool_iterations: max_iter,
auto_approve_tools: true,
default_timezone: "UTC".to_string(),
},
deps,
Arc::new(ChannelManager::new()),
@@ -2121,96 +1899,4 @@ mod tests {
}
}
}
#[test]
fn test_strip_internal_tool_call_text_removes_markers() {
let input = "[Called tool search({\"query\": \"test\"})]\nHere is the answer.";
let result = super::strip_internal_tool_call_text(input);
assert_eq!(result, "Here is the answer.");
}
#[test]
fn test_strip_internal_tool_call_text_removes_returned_markers() {
let input = "[Tool search returned: some result]\nSummary of findings.";
let result = super::strip_internal_tool_call_text(input);
assert_eq!(result, "Summary of findings.");
}
#[test]
fn test_strip_internal_tool_call_text_all_markers_yields_fallback() {
let input = "[Called tool search({\"query\": \"test\"})]\n[Tool search returned: error]";
let result = super::strip_internal_tool_call_text(input);
assert!(result.contains("wasn't able to complete"));
}
#[test]
fn test_strip_internal_tool_call_text_preserves_normal_text() {
let input = "This is a normal response with [brackets] inside.";
let result = super::strip_internal_tool_call_text(input);
assert_eq!(result, input);
}
#[test]
fn test_tool_error_format_includes_tool_name() {
// Regression test for issue #487: tool errors sent to the LLM should
// include the tool name so the model can reason about which tool failed
// and try alternatives.
let tool_name = "http";
let err = crate::error::ToolError::ExecutionFailed {
name: tool_name.to_string(),
reason: "connection refused".to_string(),
};
let formatted = format!("Tool '{}' failed: {}", tool_name, err);
assert!(
formatted.contains("Tool 'http' failed:"),
"Error should identify the tool by name, got: {formatted}"
);
assert!(
formatted.contains("connection refused"),
"Error should include the underlying reason, got: {formatted}"
);
}
#[test]
fn test_image_sentinel_empty_data_url_should_be_skipped() {
// Regression: unwrap_or_default() on missing "data" field produces an empty
// string. Broadcasting an empty data_url would send a broken SSE event.
let sentinel = serde_json::json!({
"type": "image_generated",
"path": "/tmp/image.png"
// "data" field is missing
});
let data_url = sentinel
.get("data")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
assert!(
data_url.is_empty(),
"Missing 'data' field should produce empty string"
);
// The fix: empty data_url means we skip broadcasting
}
#[test]
fn test_image_sentinel_present_data_url_is_valid() {
let sentinel = serde_json::json!({
"type": "image_generated",
"data": "data:image/png;base64,abc123",
"path": "/tmp/image.png"
});
let data_url = sentinel
.get("data")
.and_then(|v| v.as_str())
.unwrap_or_default()
.to_string();
assert!(
!data_url.is_empty(),
"Present 'data' field should produce non-empty string"
);
}
}
+8 -169
View File
@@ -29,8 +29,8 @@ use std::time::Duration;
use tokio::sync::mpsc;
use crate::channels::OutgoingResponse;
use crate::db::Database;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
use crate::safety::SafetyLayer;
use crate::workspace::Workspace;
use crate::workspace::hygiene::HygieneConfig;
@@ -47,12 +47,6 @@ pub struct HeartbeatConfig {
pub notify_user_id: Option<String>,
/// Channel to notify on heartbeat findings.
pub notify_channel: Option<String>,
/// Hour (0-23) when quiet hours start.
pub quiet_hours_start: Option<u32>,
/// Hour (0-23) when quiet hours end.
pub quiet_hours_end: Option<u32>,
/// Timezone for quiet hours evaluation (IANA name).
pub timezone: Option<String>,
}
impl Default for HeartbeatConfig {
@@ -63,9 +57,6 @@ impl Default for HeartbeatConfig {
max_failures: 3,
notify_user_id: None,
notify_channel: None,
quiet_hours_start: None,
quiet_hours_end: None,
timezone: None,
}
}
}
@@ -83,26 +74,6 @@ impl HeartbeatConfig {
self
}
/// Check whether the current time falls within configured quiet hours.
pub fn is_quiet_hours(&self) -> bool {
use chrono::Timelike;
let (Some(start), Some(end)) = (self.quiet_hours_start, self.quiet_hours_end) else {
return false;
};
let tz = self
.timezone
.as_deref()
.and_then(crate::timezone::parse_timezone)
.unwrap_or(chrono_tz::UTC);
let now_hour = crate::timezone::now_in_tz(tz).hour();
if start <= end {
now_hour >= start && now_hour < end
} else {
// Wraps midnight, e.g. 22..06
now_hour >= start || now_hour < end
}
}
/// Set the notification target.
pub fn with_notify(mut self, user_id: impl Into<String>, channel: impl Into<String>) -> Self {
self.notify_user_id = Some(user_id.into());
@@ -130,8 +101,8 @@ pub struct HeartbeatRunner {
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
store: Option<Arc<dyn Database>>,
consecutive_failures: u32,
}
@@ -142,14 +113,15 @@ impl HeartbeatRunner {
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
) -> Self {
Self {
config,
hygiene_config,
workspace,
llm,
safety,
response_tx: None,
store: None,
consecutive_failures: 0,
}
}
@@ -160,12 +132,6 @@ impl HeartbeatRunner {
self
}
/// Set the database store for persistent heartbeat conversations.
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
self.store = Some(store);
self
}
/// Run the heartbeat loop.
///
/// This runs forever, checking periodically based on the configured interval.
@@ -187,12 +153,6 @@ impl HeartbeatRunner {
loop {
interval.tick().await;
// Skip during quiet hours
if self.config.is_quiet_hours() {
tracing::debug!("Heartbeat skipped: quiet hours");
continue;
}
// Run memory hygiene in the background so it never delays the
// heartbeat checklist. Failures are logged inside run_if_due.
let hygiene_workspace = Arc::clone(&self.workspace);
@@ -204,7 +164,6 @@ impl HeartbeatRunner {
if report.had_work() {
tracing::info!(
daily_logs_deleted = report.daily_logs_deleted,
conversation_docs_deleted = report.conversation_docs_deleted,
"heartbeat: memory hygiene deleted stale documents"
);
}
@@ -303,7 +262,7 @@ impl HeartbeatRunner {
.with_max_tokens(max_tokens)
.with_temperature(0.3);
let reasoning = Reasoning::new(self.llm.clone());
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
let (content, _usage) = match reasoning.complete(request).await {
Ok(r) => r,
Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)),
@@ -332,32 +291,9 @@ impl HeartbeatRunner {
return;
};
let user_id = self.config.notify_user_id.as_deref().unwrap_or("default");
// Persist to heartbeat conversation and get thread_id
let thread_id = if let Some(ref store) = self.store {
match store.get_or_create_heartbeat_conversation(user_id).await {
Ok(conv_id) => {
if let Err(e) = store
.add_conversation_message(conv_id, "assistant", message)
.await
{
tracing::error!("Failed to persist heartbeat message: {}", e);
}
Some(conv_id.to_string())
}
Err(e) => {
tracing::error!("Failed to get heartbeat conversation: {}", e);
None
}
}
} else {
None
};
let response = OutgoingResponse {
content: format!("🔔 *Heartbeat Alert*\n\n{}", message),
thread_id,
thread_id: None,
attachments: Vec::new(),
metadata: serde_json::json!({
"source": "heartbeat",
@@ -417,16 +353,13 @@ pub fn spawn_heartbeat(
hygiene_config: HygieneConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
safety: Arc<SafetyLayer>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
store: Option<Arc<dyn Database>>,
) -> tokio::task::JoinHandle<()> {
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm);
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm, safety);
if let Some(tx) = response_tx {
runner = runner.with_response_channel(tx);
}
if let Some(s) = store {
runner = runner.with_store(s);
}
tokio::spawn(async move {
runner.run().await;
@@ -561,98 +494,4 @@ mod tests {
let content = "<!-- comment -->\nActual task here";
assert!(!is_effectively_empty(content));
}
// ==================== quiet hours ====================
#[test]
fn test_quiet_hours_inside() {
use chrono::{Timelike, Utc};
let now_utc = Utc::now();
let hour = now_utc.hour();
let start = hour;
let end = (hour + 1) % 24;
let config = HeartbeatConfig {
quiet_hours_start: Some(start),
quiet_hours_end: Some(end),
timezone: Some("UTC".to_string()),
..HeartbeatConfig::default()
};
// Current UTC hour is inside [start, end) by construction
assert!(config.is_quiet_hours());
}
#[test]
fn test_quiet_hours_outside() {
use chrono::{Timelike, Utc};
let now_utc = Utc::now();
let hour = now_utc.hour();
let start = (hour + 1) % 24;
let end = (hour + 2) % 24;
let config = HeartbeatConfig {
quiet_hours_start: Some(start),
quiet_hours_end: Some(end),
timezone: Some("UTC".to_string()),
..HeartbeatConfig::default()
};
// Current UTC hour is outside [start, end) by construction
assert!(!config.is_quiet_hours());
}
#[test]
fn test_quiet_hours_wraparound_excludes_now() {
use chrono::{Timelike, Utc};
let now_utc = Utc::now();
let hour = now_utc.hour();
// Window covers all hours except the current one
let start = (hour + 1) % 24;
let end = hour;
let config = HeartbeatConfig {
quiet_hours_start: Some(start),
quiet_hours_end: Some(end),
timezone: Some("UTC".to_string()),
..HeartbeatConfig::default()
};
assert!(!config.is_quiet_hours());
}
#[test]
fn test_quiet_hours_none_configured() {
let config = HeartbeatConfig::default();
assert!(!config.is_quiet_hours());
}
#[test]
fn test_quiet_hours_same_start_end() {
let config = HeartbeatConfig {
quiet_hours_start: Some(10),
quiet_hours_end: Some(10),
timezone: Some("UTC".to_string()),
..HeartbeatConfig::default()
};
// start == end means zero-width window, should be false
assert!(!config.is_quiet_hours());
}
#[test]
fn test_spawn_heartbeat_accepts_store_param() {
// Regression: spawn_heartbeat must accept an optional Database store
// for persisting heartbeat notifications to a dedicated conversation.
// Compile-time check: the 7th parameter is `Option<Arc<dyn Database>>`.
#[allow(clippy::type_complexity)]
let _fn_ptr: fn(
HeartbeatConfig,
HygieneConfig,
Arc<crate::workspace::Workspace>,
Arc<dyn crate::llm::LlmProvider>,
Option<tokio::sync::mpsc::Sender<crate::channels::OutgoingResponse>>,
Option<Arc<dyn crate::db::Database>>,
) -> tokio::task::JoinHandle<()> = spawn_heartbeat;
let _ = _fn_ptr;
}
}
-1
View File
@@ -11,7 +11,6 @@
//! - Context compaction for long conversations
mod agent_loop;
mod attachments;
mod commands;
pub mod compaction;
pub mod context_monitor;
+11 -112
View File
@@ -57,11 +57,7 @@ pub struct Routine {
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Trigger {
/// Fire on a cron schedule (e.g. "0 9 * * MON-FRI" or "every 2h").
Cron {
schedule: String,
#[serde(default)]
timezone: Option<String>,
},
Cron { schedule: String },
/// Fire when a channel message matches a pattern.
Event {
/// Optional channel filter (e.g. "telegram", "slack").
@@ -103,21 +99,7 @@ impl Trigger {
field: "schedule".into(),
})?
.to_string();
let timezone = config
.get("timezone")
.and_then(|v| v.as_str())
.and_then(|tz| {
if crate::timezone::parse_timezone(tz).is_some() {
Some(tz.to_string())
} else {
tracing::warn!(
"Ignoring invalid timezone '{}' from DB for cron trigger",
tz
);
None
}
});
Ok(Trigger::Cron { schedule, timezone })
Ok(Trigger::Cron { schedule })
}
"event" => {
let pattern = config
@@ -155,10 +137,7 @@ impl Trigger {
/// Serialize trigger-specific config to JSON for DB storage.
pub fn to_config_json(&self) -> serde_json::Value {
match self {
Trigger::Cron { schedule, timezone } => serde_json::json!({
"schedule": schedule,
"timezone": timezone,
}),
Trigger::Cron { schedule } => serde_json::json!({ "schedule": schedule }),
Trigger::Event { channel, pattern } => serde_json::json!({
"pattern": pattern,
"channel": channel,
@@ -196,11 +175,6 @@ pub enum RoutineAction {
/// Max reasoning iterations (default: 10).
#[serde(default = "default_max_iterations")]
max_iterations: u32,
/// Tool names pre-authorized for `Always`-approval tools (e.g. destructive
/// shell commands, cross-channel messaging). `UnlessAutoApproved` tools are
/// automatically permitted in routine jobs without listing them here.
#[serde(default)]
tool_permissions: Vec<String>,
},
}
@@ -212,19 +186,6 @@ fn default_max_iterations() -> u32 {
10
}
/// Parse a `tool_permissions` JSON array into a `Vec<String>`.
pub fn parse_tool_permissions(value: &serde_json::Value) -> Vec<String> {
value
.get("tool_permissions")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default()
}
impl RoutineAction {
/// The string tag stored in the DB action_type column.
pub fn type_tag(&self) -> &'static str {
@@ -287,12 +248,10 @@ impl RoutineAction {
.and_then(|v| v.as_u64())
.unwrap_or(default_max_iterations() as u64)
as u32;
let tool_permissions = parse_tool_permissions(&config);
Ok(RoutineAction::FullJob {
title,
description,
max_iterations,
tool_permissions,
})
}
other => Err(RoutineError::UnknownActionType {
@@ -317,12 +276,10 @@ impl RoutineAction {
title,
description,
max_iterations,
tool_permissions,
} => serde_json::json!({
"title": title,
"description": description,
"max_iterations": max_iterations,
"tool_permissions": tool_permissions,
}),
}
}
@@ -436,25 +393,12 @@ pub fn content_hash(content: &str) -> u64 {
}
/// Parse a cron expression and compute the next fire time from now.
///
/// When `timezone` is provided and valid, the schedule is evaluated in that
/// timezone and the result is converted back to UTC. Otherwise UTC is used.
pub fn next_cron_fire(
schedule: &str,
timezone: Option<&str>,
) -> Result<Option<DateTime<Utc>>, RoutineError> {
pub fn next_cron_fire(schedule: &str) -> Result<Option<DateTime<Utc>>, RoutineError> {
let cron_schedule =
cron::Schedule::from_str(schedule).map_err(|e| RoutineError::InvalidCron {
reason: e.to_string(),
})?;
if let Some(tz) = timezone.and_then(crate::timezone::parse_timezone) {
Ok(cron_schedule
.upcoming(tz)
.next()
.map(|dt| dt.with_timezone(&Utc)))
} else {
Ok(cron_schedule.upcoming(Utc).next())
}
Ok(cron_schedule.upcoming(Utc).next())
}
#[cfg(test)]
@@ -467,11 +411,10 @@ mod tests {
fn test_trigger_roundtrip() {
let trigger = Trigger::Cron {
schedule: "0 9 * * MON-FRI".to_string(),
timezone: None,
};
let json = trigger.to_config_json();
let parsed = Trigger::from_db("cron", json).expect("parse cron");
assert!(matches!(parsed, Trigger::Cron { schedule, .. } if schedule == "0 9 * * MON-FRI"));
assert!(matches!(parsed, Trigger::Cron { schedule } if schedule == "0 9 * * MON-FRI"));
}
#[test]
@@ -507,13 +450,12 @@ mod tests {
title: "Deploy review".to_string(),
description: "Review and deploy pending changes".to_string(),
max_iterations: 5,
tool_permissions: vec!["shell".to_string()],
};
let json = action.to_config_json();
let parsed = RoutineAction::from_db("full_job", json).expect("parse full_job");
assert!(
matches!(parsed, RoutineAction::FullJob { title, max_iterations, tool_permissions, .. }
if title == "Deploy review" && max_iterations == 5 && tool_permissions == vec!["shell".to_string()])
matches!(parsed, RoutineAction::FullJob { title, max_iterations, .. }
if title == "Deploy review" && max_iterations == 5)
);
}
@@ -544,58 +486,16 @@ mod tests {
#[test]
fn test_next_cron_fire_valid() {
// Every minute should always have a next fire
let next = next_cron_fire("* * * * * *", None).expect("valid cron");
let next = next_cron_fire("* * * * * *").expect("valid cron");
assert!(next.is_some());
}
#[test]
fn test_next_cron_fire_invalid() {
let result = next_cron_fire("not a cron", None);
let result = next_cron_fire("not a cron");
assert!(result.is_err());
}
#[test]
fn test_trigger_cron_timezone_roundtrip() {
let trigger = Trigger::Cron {
schedule: "0 9 * * MON-FRI".to_string(),
timezone: Some("America/New_York".to_string()),
};
let json = trigger.to_config_json();
let parsed = Trigger::from_db("cron", json).expect("parse cron");
assert!(matches!(parsed, Trigger::Cron { schedule, timezone }
if schedule == "0 9 * * MON-FRI"
&& timezone.as_deref() == Some("America/New_York")));
}
#[test]
fn test_trigger_cron_no_timezone_backward_compat() {
let json = serde_json::json!({"schedule": "0 9 * * *"});
let parsed = Trigger::from_db("cron", json).expect("parse cron");
assert!(matches!(parsed, Trigger::Cron { timezone, .. } if timezone.is_none()));
}
#[test]
fn test_trigger_cron_invalid_timezone_coerced_to_none() {
let json = serde_json::json!({"schedule": "0 9 * * *", "timezone": "Fake/Zone"});
let parsed = Trigger::from_db("cron", json).expect("parse cron");
assert!(
matches!(parsed, Trigger::Cron { timezone, .. } if timezone.is_none()),
"invalid timezone should be coerced to None"
);
}
#[test]
fn test_next_cron_fire_with_timezone() {
let next_utc = next_cron_fire("0 0 9 * * * *", None)
.expect("valid cron")
.expect("has next");
let next_est = next_cron_fire("0 0 9 * * * *", Some("America/New_York"))
.expect("valid cron")
.expect("has next");
// EST is UTC-5 (or EDT UTC-4), so the UTC result should differ
assert_ne!(next_utc, next_est, "timezone should shift the fire time");
}
#[test]
fn test_guardrails_default() {
let g = RoutineGuardrails::default();
@@ -608,8 +508,7 @@ mod tests {
fn test_trigger_type_tag() {
assert_eq!(
Trigger::Cron {
schedule: String::new(),
timezone: None,
schedule: String::new()
}
.type_tag(),
"cron"
+8 -91
View File
@@ -28,7 +28,6 @@ use crate::config::RoutineConfig;
use crate::db::Database;
use crate::error::RoutineError;
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
use crate::tools::ApprovalContext;
use crate::workspace::Workspace;
/// The routine execution engine.
@@ -170,7 +169,7 @@ impl RoutineEngine {
continue;
}
let detail = if let Trigger::Cron { ref schedule, .. } = routine.trigger {
let detail = if let Trigger::Cron { ref schedule } = routine.trigger {
Some(schedule.clone())
} else {
None
@@ -181,14 +180,7 @@ impl RoutineEngine {
}
/// Fire a routine manually (from tool call or CLI).
///
/// Bypasses cooldown checks (those only apply to cron/event triggers).
/// Still enforces enabled check and concurrent run limit.
pub async fn fire_manual(
&self,
routine_id: Uuid,
user_id: Option<&str>,
) -> Result<Uuid, RoutineError> {
pub async fn fire_manual(&self, routine_id: Uuid) -> Result<Uuid, RoutineError> {
let routine = self
.store
.get_routine(routine_id)
@@ -198,13 +190,6 @@ impl RoutineEngine {
})?
.ok_or(RoutineError::NotFound { id: routine_id })?;
// Enforce ownership when a user_id is provided (gateway calls).
if let Some(uid) = user_id
&& routine.user_id != uid
{
return Err(RoutineError::NotAuthorized { id: routine_id });
}
if !routine.enabled {
return Err(RoutineError::Disabled {
name: routine.name.clone(),
@@ -342,19 +327,7 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
title,
description,
max_iterations,
tool_permissions,
} => {
execute_full_job(
&ctx,
&routine,
&run,
title,
description,
*max_iterations,
tool_permissions,
)
.await
}
} => execute_full_job(&ctx, &routine, &run, title, description, *max_iterations).await,
};
// Decrement running count
@@ -380,12 +353,8 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
// Update routine runtime state
let now = Utc::now();
let next_fire = if let Trigger::Cron {
ref schedule,
ref timezone,
} = routine.trigger
{
next_cron_fire(schedule, timezone.as_deref()).unwrap_or(None)
let next_fire = if let Trigger::Cron { ref schedule } = routine.trigger {
next_cron_fire(schedule).unwrap_or(None)
} else {
None
};
@@ -411,39 +380,6 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
tracing::error!(routine = %routine.name, "Failed to update runtime state: {}", e);
}
// Persist routine result to its dedicated conversation thread
let thread_id = match ctx
.store
.get_or_create_routine_conversation(routine.id, &routine.name, &routine.user_id)
.await
{
Ok(conv_id) => {
tracing::debug!(
routine = %routine.name,
routine_id = %routine.id,
conversation_id = %conv_id,
"Resolved routine conversation thread"
);
// Record the run result as a conversation message
let msg = match (&summary, status) {
(Some(s), _) => format!("[{}] {}: {}", run.trigger_type, status, s),
(None, _) => format!("[{}] {}", run.trigger_type, status),
};
if let Err(e) = ctx
.store
.add_conversation_message(conv_id, "assistant", &msg)
.await
{
tracing::error!(routine = %routine.name, "Failed to persist routine message: {}", e);
}
Some(conv_id.to_string())
}
Err(e) => {
tracing::error!(routine = %routine.name, "Failed to get routine conversation: {}", e);
None
}
};
// Send notifications based on config
send_notification(
&ctx.notify_tx,
@@ -451,7 +387,6 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
&routine.name,
status,
summary.as_deref(),
thread_id.as_deref(),
)
.await;
}
@@ -483,7 +418,6 @@ async fn execute_full_job(
title: &str,
description: &str,
max_iterations: u32,
tool_permissions: &[String],
) -> Result<(RunStatus, Option<String>, Option<i32>), RoutineError> {
let scheduler = ctx
.scheduler
@@ -492,26 +426,10 @@ async fn execute_full_job(
reason: "scheduler not available".to_string(),
})?;
let mut metadata = serde_json::json!({ "max_iterations": max_iterations });
// Carry the routine's notify config in job metadata so the message tool
// can resolve channel/target per-job without global state mutation.
if let Some(channel) = &routine.notify.channel {
metadata["notify_channel"] = serde_json::json!(channel);
}
metadata["notify_user"] = serde_json::json!(&routine.notify.user);
// Build approval context: UnlessAutoApproved tools are auto-approved for routines;
// Always tools require explicit listing in tool_permissions.
let approval_context = ApprovalContext::autonomous_with_tools(tool_permissions.iter().cloned());
let metadata = serde_json::json!({ "max_iterations": max_iterations });
let job_id = scheduler
.dispatch_job_with_context(
&routine.user_id,
title,
description,
Some(metadata),
approval_context,
)
.dispatch_job(&routine.user_id, title, description, Some(metadata))
.await
.map_err(|e| RoutineError::JobDispatchFailed {
reason: format!("failed to dispatch job: {e}"),
@@ -655,7 +573,6 @@ async fn send_notification(
routine_name: &str,
status: RunStatus,
summary: Option<&str>,
thread_id: Option<&str>,
) {
let should_notify = match status {
RunStatus::Ok => notify.on_success,
@@ -682,7 +599,7 @@ async fn send_notification(
let response = OutgoingResponse {
content: message,
thread_id: thread_id.map(String::from),
thread_id: None,
attachments: Vec::new(),
metadata: serde_json::json!({
"source": "routine",
+3 -302
View File
@@ -10,7 +10,6 @@ use uuid::Uuid;
use crate::agent::task::{Task, TaskContext, TaskOutput};
use crate::agent::worker::{Worker, WorkerDeps};
use crate::channels::web::types::SseEvent;
use crate::config::AgentConfig;
use crate::context::{ContextManager, JobContext, JobState};
use crate::db::Database;
@@ -18,7 +17,7 @@ use crate::error::{Error, JobError};
use crate::hooks::HookRegistry;
use crate::llm::LlmProvider;
use crate::safety::SafetyLayer;
use crate::tools::{ApprovalContext, ToolRegistry};
use crate::tools::ToolRegistry;
/// Message to send to a worker.
#[derive(Debug)]
@@ -29,8 +28,6 @@ pub enum WorkerMessage {
Stop,
/// Check health.
Ping,
/// Inject a follow-up user message into the worker's reasoning context.
UserMessage(String),
}
/// Status of a scheduled job.
@@ -54,10 +51,6 @@ pub struct Scheduler {
tools: Arc<ToolRegistry>,
store: Option<Arc<dyn Database>>,
hooks: Arc<HookRegistry>,
/// SSE broadcast sender for live job event streaming.
sse_tx: Option<tokio::sync::broadcast::Sender<SseEvent>>,
/// HTTP interceptor for trace recording/replay (propagated to workers).
http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>,
/// Running jobs (main LLM-driven jobs).
jobs: Arc<RwLock<HashMap<Uuid, ScheduledJob>>>,
/// Running sub-tasks (tool executions, background tasks).
@@ -83,26 +76,11 @@ impl Scheduler {
tools,
store,
hooks,
sse_tx: None,
http_interceptor: None,
jobs: Arc::new(RwLock::new(HashMap::new())),
subtasks: Arc::new(RwLock::new(HashMap::new())),
}
}
/// Set the SSE broadcast sender for live job event streaming.
pub fn set_sse_sender(&mut self, tx: tokio::sync::broadcast::Sender<SseEvent>) {
self.sse_tx = Some(tx);
}
/// Set the HTTP interceptor for trace recording/replay.
pub fn set_http_interceptor(
&mut self,
interceptor: Arc<dyn crate::llm::recording::HttpInterceptor>,
) {
self.http_interceptor = Some(interceptor);
}
/// Create, persist, and schedule a job in one shot.
///
/// This is the preferred entry point for dispatching new jobs. It:
@@ -119,41 +97,6 @@ impl Scheduler {
title: &str,
description: &str,
metadata: Option<serde_json::Value>,
) -> Result<Uuid, JobError> {
self.dispatch_job_inner(user_id, title, description, metadata, None)
.await
}
/// Dispatch a job with an explicit approval context for autonomous execution.
///
/// Same as `dispatch_job`, but the worker will use the given `ApprovalContext`
/// to determine which tools are pre-approved (instead of blocking all non-`Never` tools).
pub async fn dispatch_job_with_context(
&self,
user_id: &str,
title: &str,
description: &str,
metadata: Option<serde_json::Value>,
approval_context: ApprovalContext,
) -> Result<Uuid, JobError> {
self.dispatch_job_inner(
user_id,
title,
description,
metadata,
Some(approval_context),
)
.await
}
/// Shared implementation for `dispatch_job` and `dispatch_job_with_context`.
async fn dispatch_job_inner(
&self,
user_id: &str,
title: &str,
description: &str,
metadata: Option<serde_json::Value>,
approval_context: Option<ApprovalContext>,
) -> Result<Uuid, JobError> {
let job_id = self
.context_manager
@@ -178,21 +121,12 @@ impl Scheduler {
})?;
}
self.schedule_with_context(job_id, approval_context).await?;
self.schedule(job_id).await?;
Ok(job_id)
}
/// Schedule a job for execution.
pub async fn schedule(&self, job_id: Uuid) -> Result<(), JobError> {
self.schedule_with_context(job_id, None).await
}
/// Schedule a job with an optional approval context.
async fn schedule_with_context(
&self,
job_id: Uuid,
approval_context: Option<ApprovalContext>,
) -> Result<(), JobError> {
// Hold write lock for the entire check-insert sequence to prevent
// TOCTOU races where two concurrent calls both pass the checks.
{
@@ -235,9 +169,6 @@ impl Scheduler {
hooks: self.hooks.clone(),
timeout: self.config.job_timeout,
use_planning: self.config.use_planning,
sse_tx: self.sse_tx.clone(),
approval_context,
http_interceptor: self.http_interceptor.clone(),
};
let worker = Worker::new(job_id, deps);
@@ -314,14 +245,11 @@ impl Scheduler {
let context_manager = self.context_manager.clone();
let safety = self.safety.clone();
// TODO: propagate parent job's ApprovalContext here when subtasks
// are used in autonomous/routine paths (currently only used in tests).
tokio::spawn(async move {
let result = Self::execute_tool_task(
tools,
context_manager,
safety,
None,
tool_parent_id,
&tool_name,
params,
@@ -450,7 +378,6 @@ impl Scheduler {
tools: Arc<ToolRegistry>,
context_manager: Arc<ContextManager>,
safety: Arc<SafetyLayer>,
approval_context: Option<ApprovalContext>,
job_id: Uuid,
tool_name: &str,
params: serde_json::Value,
@@ -474,10 +401,7 @@ impl Scheduler {
.into());
}
let requirement = tool.requires_approval(&params);
let blocked =
ApprovalContext::is_blocked_or_default(&approval_context, tool_name, requirement);
if blocked {
if tool.requires_approval(&params).is_required() {
return Err(crate::error::ToolError::AuthRequired {
name: tool_name.to_string(),
}
@@ -576,26 +500,6 @@ impl Scheduler {
Ok(())
}
/// Send a follow-up user message to a running job.
///
/// Returns `Ok(())` if the message was queued, `Err` if the job is not running.
pub async fn send_message(&self, job_id: Uuid, content: String) -> Result<(), JobError> {
// Clone the sender while holding the lock, then release before the
// async send to avoid blocking scheduler writes during backpressure.
let tx = {
let jobs = self.jobs.read().await;
let scheduled = jobs.get(&job_id).ok_or(JobError::NotFound { id: job_id })?;
scheduled.tx.clone()
};
tx.send(WorkerMessage::UserMessage(content))
.await
.map_err(|_| JobError::Failed {
id: job_id,
reason: "Worker channel closed".to_string(),
})?;
Ok(())
}
/// Check if a job is running.
pub async fn is_running(&self, job_id: Uuid) -> bool {
self.jobs.read().await.contains_key(&job_id)
@@ -681,11 +585,6 @@ impl Scheduler {
#[cfg(test)]
mod tests {
use super::*;
use crate::config::SafetyConfig;
use crate::safety::SafetyLayer;
use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput};
#[test]
fn test_scheduler_creation() {
// Would need to mock dependencies for proper testing
@@ -696,202 +595,4 @@ mod tests {
// This test would need mock dependencies.
// For now just verify the empty case doesn't panic.
}
/// A tool that returns `UnlessAutoApproved`.
struct SoftApprovalTool;
#[async_trait::async_trait]
impl Tool for SoftApprovalTool {
fn name(&self) -> &str {
"soft_gate"
}
fn description(&self) -> &str {
"needs soft approval"
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({"type": "object", "properties": {}})
}
async fn execute(
&self,
_params: serde_json::Value,
_ctx: &JobContext,
) -> Result<ToolOutput, ToolError> {
Ok(ToolOutput::text(
"soft_ok",
std::time::Instant::now().elapsed(),
))
}
fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement {
ApprovalRequirement::UnlessAutoApproved
}
fn requires_sanitization(&self) -> bool {
false
}
}
/// A tool that returns `Always`.
struct HardApprovalTool;
#[async_trait::async_trait]
impl Tool for HardApprovalTool {
fn name(&self) -> &str {
"hard_gate"
}
fn description(&self) -> &str {
"needs hard approval"
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({"type": "object", "properties": {}})
}
async fn execute(
&self,
_params: serde_json::Value,
_ctx: &JobContext,
) -> Result<ToolOutput, ToolError> {
Ok(ToolOutput::text(
"hard_ok",
std::time::Instant::now().elapsed(),
))
}
fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement {
ApprovalRequirement::Always
}
fn requires_sanitization(&self) -> bool {
false
}
}
async fn setup_tools_and_job() -> (
Arc<ToolRegistry>,
Arc<ContextManager>,
Arc<SafetyLayer>,
Uuid,
) {
let registry = ToolRegistry::new();
registry.register(Arc::new(SoftApprovalTool)).await;
registry.register(Arc::new(HardApprovalTool)).await;
let cm = Arc::new(ContextManager::new(5));
let job_id = cm.create_job("test", "approval test").await.unwrap();
cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None))
.await
.unwrap()
.unwrap();
let safety = Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
}));
(Arc::new(registry), cm, safety, job_id)
}
#[tokio::test]
async fn test_execute_tool_task_blocks_without_context() {
let (tools, cm, safety, job_id) = setup_tools_and_job().await;
// Without approval context, UnlessAutoApproved is blocked
let result = Scheduler::execute_tool_task(
tools.clone(),
cm.clone(),
safety.clone(),
None,
job_id,
"soft_gate",
serde_json::json!({}),
)
.await;
assert!(
result.is_err(),
"soft_gate should be blocked without context"
);
// Always is also blocked
let result = Scheduler::execute_tool_task(
tools,
cm,
safety,
None,
job_id,
"hard_gate",
serde_json::json!({}),
)
.await;
assert!(
result.is_err(),
"hard_gate should be blocked without context"
);
}
#[tokio::test]
async fn test_execute_tool_task_autonomous_unblocks_soft() {
let (tools, cm, safety, job_id) = setup_tools_and_job().await;
// Autonomous context auto-approves UnlessAutoApproved
let result = Scheduler::execute_tool_task(
tools.clone(),
cm.clone(),
safety.clone(),
Some(ApprovalContext::autonomous()),
job_id,
"soft_gate",
serde_json::json!({}),
)
.await;
assert!(
result.is_ok(),
"soft_gate should pass with autonomous context"
);
// But still blocks Always
let result = Scheduler::execute_tool_task(
tools,
cm,
safety,
Some(ApprovalContext::autonomous()),
job_id,
"hard_gate",
serde_json::json!({}),
)
.await;
assert!(
result.is_err(),
"hard_gate should still be blocked without explicit permission"
);
}
#[tokio::test]
async fn test_execute_tool_task_autonomous_with_permissions() {
let (tools, cm, safety, job_id) = setup_tools_and_job().await;
// Autonomous context with explicit permission for hard_gate
let ctx = ApprovalContext::autonomous_with_tools(["hard_gate".to_string()]);
let result = Scheduler::execute_tool_task(
tools.clone(),
cm.clone(),
safety.clone(),
Some(ctx.clone()),
job_id,
"soft_gate",
serde_json::json!({}),
)
.await;
assert!(result.is_ok(), "soft_gate should pass");
let result = Scheduler::execute_tool_task(
tools,
cm,
safety,
Some(ctx),
job_id,
"hard_gate",
serde_json::json!({}),
)
.await;
assert!(
result.is_ok(),
"hard_gate should pass with explicit permission"
);
}
}
+13 -349
View File
@@ -16,7 +16,6 @@ use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::channels::web::util::truncate_preview;
use crate::llm::{ChatMessage, ToolCall};
/// A session containing one or more threads.
@@ -149,12 +148,8 @@ pub struct PendingApproval {
pub request_id: Uuid,
/// Tool name requiring approval.
pub tool_name: String,
/// Tool parameters (original values, used for execution).
/// Tool parameters.
pub parameters: serde_json::Value,
/// Redacted tool parameters (sensitive values replaced with `[REDACTED]`).
/// Used for display in approval UI, logs, and SSE broadcasts.
#[serde(default)]
pub display_parameters: serde_json::Value,
/// Description of what the tool will do.
pub description: String,
/// Tool call ID from LLM (for proper context continuation).
@@ -165,10 +160,6 @@ pub struct PendingApproval {
/// executed yet when approval was requested.
#[serde(default)]
pub deferred_tool_calls: Vec<ToolCall>,
/// User timezone at the time the approval was requested, so it persists
/// through the approval flow even if the approval message lacks timezone.
#[serde(default)]
pub user_timezone: Option<String>,
}
/// A conversation thread within a session.
@@ -321,60 +312,11 @@ impl Thread {
}
}
/// Get all messages for context building, including tool call history.
///
/// Emits the full LLM-compatible message sequence per turn:
/// `user → [assistant_with_tool_calls → tool_result*] → assistant`
///
/// This ensures the LLM sees prior tool executions and won't re-attempt
/// completed actions in subsequent turns.
/// Get all messages for context building.
pub fn messages(&self) -> Vec<ChatMessage> {
let mut messages = Vec::new();
for turn in &self.turns {
if turn.image_content_parts.is_empty() {
messages.push(ChatMessage::user(&turn.user_input));
} else {
messages.push(ChatMessage::user_with_parts(
&turn.user_input,
turn.image_content_parts.clone(),
));
}
if !turn.tool_calls.is_empty() {
// Build ToolCall objects with synthetic stable IDs
let tool_calls: Vec<ToolCall> = turn
.tool_calls
.iter()
.enumerate()
.map(|(i, tc)| ToolCall {
id: format!("turn{}_{}", turn.turn_number, i),
name: tc.name.clone(),
arguments: tc.parameters.clone(),
})
.collect();
// Assistant message declaring the tool calls (no text content)
messages.push(ChatMessage::assistant_with_tool_calls(None, tool_calls));
// Individual tool result messages, truncated to limit context size.
for (i, tc) in turn.tool_calls.iter().enumerate() {
let call_id = format!("turn{}_{}", turn.turn_number, i);
let content = if let Some(ref err) = tc.error {
// .error already contains the full error text;
// pass through without wrapping to avoid double-prefix.
truncate_preview(err, 1000)
} else if let Some(ref res) = tc.result {
let raw = match res {
serde_json::Value::String(s) => s.clone(),
other => other.to_string(),
};
truncate_preview(&raw, 1000)
} else {
"OK".to_string()
};
messages.push(ChatMessage::tool_result(call_id, &tc.name, content));
}
}
messages.push(ChatMessage::user(&turn.user_input));
if let Some(ref response) = turn.response {
messages.push(ChatMessage::assistant(response));
}
@@ -396,16 +338,13 @@ impl Thread {
/// Restore thread state from a checkpoint's messages.
///
/// Clears existing turns and rebuilds from the message sequence.
/// Handles the full message pattern including tool messages:
/// `user → [assistant_with_tool_calls → tool_result*] → assistant`
///
/// Also supports the legacy pattern (user/assistant pairs only) for
/// backward compatibility with old checkpoint data.
/// Clears existing turns and rebuilds from message pairs.
/// Messages should alternate: user, assistant, user, assistant...
pub fn restore_from_messages(&mut self, messages: Vec<ChatMessage>) {
self.turns.clear();
self.state = ThreadState::Idle;
// Messages alternate: user, assistant, user, assistant...
let mut iter = messages.into_iter().peekable();
let mut turn_number = 0;
@@ -413,58 +352,18 @@ impl Thread {
if msg.role == crate::llm::Role::User {
let mut turn = Turn::new(turn_number, &msg.content);
// Consume tool call sequences (assistant_with_tool_calls + tool_results).
// A single turn may contain multiple rounds of tool calls, so we
// track the cumulative base index into turn.tool_calls.
while let Some(next) = iter.peek() {
if next.role == crate::llm::Role::Assistant && next.tool_calls.is_some() {
let call_base_idx = turn.tool_calls.len();
if let Some(assistant_msg) = iter.next()
&& let Some(ref tcs) = assistant_msg.tool_calls
{
for tc in tcs {
turn.record_tool_call(&tc.name, tc.arguments.clone());
}
}
// Consume the corresponding tool_result messages,
// indexing relative to this batch's base offset.
let mut pos = 0;
while let Some(tr) = iter.peek() {
if tr.role != crate::llm::Role::Tool {
break;
}
if let Some(tool_msg) = iter.next() {
let idx = call_base_idx + pos;
if idx < turn.tool_calls.len() {
// Store as result — the error/success distinction
// is for the live turn only; restored context just
// needs the content the LLM originally saw.
turn.tool_calls[idx].result =
Some(serde_json::Value::String(tool_msg.content.clone()));
}
}
pos += 1;
}
} else {
break;
// Check if next is assistant response
if let Some(next) = iter.peek()
&& next.role == crate::llm::Role::Assistant
{
// iter.next() is guaranteed Some after a successful peek()
if let Some(response) = iter.next() {
turn.complete(&response.content);
}
}
// Check if next is the final assistant response for this turn
let is_final_assistant = iter.peek().is_some_and(|n| {
n.role == crate::llm::Role::Assistant && n.tool_calls.is_none()
});
if is_final_assistant && let Some(response) = iter.next() {
turn.complete(&response.content);
}
self.turns.push(turn);
turn_number += 1;
} else {
// Skip non-user messages that aren't anchored to a turn
continue;
}
}
@@ -504,11 +403,6 @@ pub struct Turn {
pub completed_at: Option<DateTime<Utc>>,
/// Error message (if failed).
pub error: Option<String>,
/// Transient image content parts for multimodal LLM input.
/// Not serialized — images are only needed for the current LLM call.
/// The text description in `user_input` persists for compaction/context.
#[serde(skip)]
pub image_content_parts: Vec<crate::llm::ContentPart>,
}
impl Turn {
@@ -523,7 +417,6 @@ impl Turn {
started_at: Utc::now(),
completed_at: None,
error: None,
image_content_parts: Vec::new(),
}
}
@@ -532,8 +425,6 @@ impl Turn {
self.response = Some(response.into());
self.state = TurnState::Completed;
self.completed_at = Some(Utc::now());
// Free image data — only needed for the initial LLM call, not subsequent turns
self.image_content_parts.clear();
}
/// Fail this turn.
@@ -541,14 +432,12 @@ impl Turn {
self.error = Some(error.into());
self.state = TurnState::Failed;
self.completed_at = Some(Utc::now());
self.image_content_parts.clear();
}
/// Interrupt this turn.
pub fn interrupt(&mut self) {
self.state = TurnState::Interrupted;
self.completed_at = Some(Utc::now());
self.image_content_parts.clear();
}
/// Record a tool call.
@@ -1061,12 +950,10 @@ mod tests {
request_id: Uuid::new_v4(),
tool_name: "shell".to_string(),
parameters: serde_json::json!({"command": "rm -rf /"}),
display_parameters: serde_json::json!({"command": "rm -rf /"}),
description: "dangerous command".to_string(),
tool_call_id: "call_123".to_string(),
context_messages: vec![ChatMessage::user("do it")],
deferred_tool_calls: vec![],
user_timezone: None,
};
thread.await_approval(approval);
@@ -1087,12 +974,10 @@ mod tests {
request_id: Uuid::new_v4(),
tool_name: "http".to_string(),
parameters: serde_json::json!({}),
display_parameters: serde_json::json!({}),
description: "test".to_string(),
tool_call_id: "call_456".to_string(),
context_messages: vec![],
deferred_tool_calls: vec![],
user_timezone: None,
};
thread.await_approval(approval);
@@ -1121,225 +1006,4 @@ mod tests {
ThreadState::Processing
);
}
// Regression tests for #568: tool call history must survive hydration.
#[test]
fn test_messages_includes_tool_calls() {
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("Search for X");
{
let turn = thread.turns.last_mut().unwrap();
turn.record_tool_call("memory_search", serde_json::json!({"query": "X"}));
turn.record_tool_result(serde_json::json!("Found X in doc.md"));
}
thread.complete_turn("I found X in doc.md.");
let messages = thread.messages();
// user + assistant_with_tool_calls + tool_result + assistant = 4
assert_eq!(messages.len(), 4);
assert_eq!(messages[0].role, crate::llm::Role::User);
assert_eq!(messages[0].content, "Search for X");
assert_eq!(messages[1].role, crate::llm::Role::Assistant);
assert!(messages[1].tool_calls.is_some());
let tcs = messages[1].tool_calls.as_ref().unwrap();
assert_eq!(tcs.len(), 1);
assert_eq!(tcs[0].name, "memory_search");
assert_eq!(messages[2].role, crate::llm::Role::Tool);
assert!(messages[2].content.contains("Found X"));
assert_eq!(messages[3].role, crate::llm::Role::Assistant);
assert_eq!(messages[3].content, "I found X in doc.md.");
}
#[test]
fn test_messages_multiple_tool_calls_per_turn() {
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("Do two things");
{
let turn = thread.turns.last_mut().unwrap();
turn.record_tool_call("echo", serde_json::json!({"msg": "a"}));
turn.record_tool_result(serde_json::json!("a"));
turn.record_tool_call("time", serde_json::json!({}));
turn.record_tool_error("timeout");
}
thread.complete_turn("Done.");
let messages = thread.messages();
// user + assistant_with_calls(2) + tool_result + tool_result + assistant = 5
assert_eq!(messages.len(), 5);
let tcs = messages[1].tool_calls.as_ref().unwrap();
assert_eq!(tcs.len(), 2);
// First tool: success
assert_eq!(messages[2].content, "a");
// Second tool: error (passed through directly, no wrapping)
assert!(messages[3].content.contains("timeout"));
}
#[test]
fn test_restore_from_messages_with_tool_calls() {
let mut thread = Thread::new(Uuid::new_v4());
// Build a message sequence with tool calls
let tc = ToolCall {
id: "call_0".to_string(),
name: "search".to_string(),
arguments: serde_json::json!({"q": "test"}),
};
let messages = vec![
ChatMessage::user("Find test"),
ChatMessage::assistant_with_tool_calls(None, vec![tc]),
ChatMessage::tool_result("call_0", "search", "result: found"),
ChatMessage::assistant("Found it."),
];
thread.restore_from_messages(messages);
assert_eq!(thread.turns.len(), 1);
let turn = &thread.turns[0];
assert_eq!(turn.user_input, "Find test");
assert_eq!(turn.tool_calls.len(), 1);
assert_eq!(turn.tool_calls[0].name, "search");
assert_eq!(
turn.tool_calls[0].result,
Some(serde_json::Value::String("result: found".to_string()))
);
assert_eq!(turn.response, Some("Found it.".to_string()));
}
#[test]
fn test_restore_from_messages_with_tool_error() {
let mut thread = Thread::new(Uuid::new_v4());
let tc = ToolCall {
id: "call_0".to_string(),
name: "http".to_string(),
arguments: serde_json::json!({}),
};
let messages = vec![
ChatMessage::user("Fetch URL"),
ChatMessage::assistant_with_tool_calls(None, vec![tc]),
ChatMessage::tool_result("call_0", "http", "Error: timeout"),
ChatMessage::assistant("The request timed out."),
];
thread.restore_from_messages(messages);
// restore_from_messages stores all tool content as result (not error),
// because it can't reliably distinguish errors from results that happen
// to start with "Error: ". The content is preserved for LLM context.
let turn = &thread.turns[0];
assert_eq!(
turn.tool_calls[0].result,
Some(serde_json::Value::String("Error: timeout".to_string()))
);
}
#[test]
fn test_messages_round_trip_with_tools() {
// Build a thread with tool calls, get messages(), restore, get messages() again
// The two message sequences should be equivalent.
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("Do search");
{
let turn = thread.turns.last_mut().unwrap();
turn.record_tool_call("search", serde_json::json!({"q": "test"}));
turn.record_tool_result(serde_json::json!("found"));
}
thread.complete_turn("Here are results.");
let messages_original = thread.messages();
// Restore into a new thread
let mut thread2 = Thread::new(Uuid::new_v4());
thread2.restore_from_messages(messages_original.clone());
let messages_restored = thread2.messages();
// Same number of messages
assert_eq!(messages_original.len(), messages_restored.len());
// Same roles
for (orig, rest) in messages_original.iter().zip(messages_restored.iter()) {
assert_eq!(orig.role, rest.role);
}
// Same final response
assert_eq!(
messages_original.last().unwrap().content,
messages_restored.last().unwrap().content
);
}
#[test]
fn test_restore_multi_stage_tool_calls() {
let mut thread = Thread::new(Uuid::new_v4());
let tc1 = ToolCall {
id: "call_a".to_string(),
name: "search".to_string(),
arguments: serde_json::json!({"q": "data"}),
};
let tc2 = ToolCall {
id: "call_b".to_string(),
name: "write".to_string(),
arguments: serde_json::json!({"path": "out.txt"}),
};
let messages = vec![
ChatMessage::user("Find and save"),
ChatMessage::assistant_with_tool_calls(None, vec![tc1]),
ChatMessage::tool_result("call_a", "search", "found data"),
ChatMessage::assistant_with_tool_calls(None, vec![tc2]),
ChatMessage::tool_result("call_b", "write", "written"),
ChatMessage::assistant("Done, saved to out.txt"),
];
thread.restore_from_messages(messages);
assert_eq!(thread.turns.len(), 1);
let turn = &thread.turns[0];
assert_eq!(turn.tool_calls.len(), 2);
assert_eq!(turn.tool_calls[0].name, "search");
assert_eq!(turn.tool_calls[1].name, "write");
assert_eq!(
turn.tool_calls[0].result,
Some(serde_json::Value::String("found data".to_string()))
);
assert_eq!(
turn.tool_calls[1].result,
Some(serde_json::Value::String("written".to_string()))
);
assert_eq!(turn.response, Some("Done, saved to out.txt".to_string()));
}
#[test]
fn test_messages_truncates_large_tool_results() {
let mut thread = Thread::new(Uuid::new_v4());
thread.start_turn("Read big file");
{
let turn = thread.turns.last_mut().unwrap();
turn.record_tool_call("read_file", serde_json::json!({"path": "big.txt"}));
let big_result = "x".repeat(2000);
turn.record_tool_result(serde_json::json!(big_result));
}
thread.complete_turn("Here's the file content.");
let messages = thread.messages();
let tool_result_content = &messages[2].content;
assert!(
tool_result_content.len() <= 1010,
"Tool result should be truncated, got {} chars",
tool_result_content.len()
);
assert!(tool_result_content.ends_with("..."));
}
}
-8
View File
@@ -14,7 +14,6 @@ impl SubmissionParser {
pub fn parse(content: &str) -> Submission {
let trimmed = content.trim();
let lower = trimmed.to_lowercase();
tracing::debug!("[SubmissionParser::parse] Parsing input: {:?}", trimmed);
// Control commands (exact match or prefix)
if lower == "/undo" {
@@ -92,13 +91,6 @@ impl SubmissionParser {
args: vec![],
};
}
if lower == "/restart" {
tracing::debug!("[SubmissionParser::parse] Recognized /restart command");
return Submission::SystemCommand {
command: "restart".to_string(),
args: vec![],
};
}
if lower.starts_with("/model") {
let args: Vec<String> = trimmed
.split_whitespace()
+83 -349
View File
@@ -20,8 +20,7 @@ use crate::channels::web::util::truncate_preview;
use crate::channels::{IncomingMessage, StatusUpdate};
use crate::context::JobContext;
use crate::error::Error;
use crate::llm::{ChatMessage, ToolCall};
use crate::tools::redact_params;
use crate::llm::ChatMessage;
impl Agent {
/// Hydrate a historical thread from DB into memory if not already present.
@@ -66,7 +65,16 @@ impl Agent {
.await
.unwrap_or_default();
msg_count = db_messages.len();
chat_messages = rebuild_chat_messages_from_db(&db_messages);
chat_messages = db_messages
.iter()
.filter_map(|m| match m.role.as_str() {
"user" => Some(ChatMessage::user(&m.content)),
"assistant" => Some(ChatMessage::assistant(&m.content)),
// tool_calls rows are UI metadata (tool name + preview),
// not part of the LLM conversation context.
_ => None,
})
.collect();
} else {
msg_count = 0;
}
@@ -221,7 +229,7 @@ impl Agent {
)
.await;
let compactor = ContextCompactor::new(self.llm().clone());
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
if let Err(e) = compactor
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
.await
@@ -248,14 +256,6 @@ impl Agent {
);
}
// Augment content with attachment context (transcripts, metadata, images)
let augmented =
crate::agent::attachments::augment_with_attachments(content, &message.attachments);
let (effective_content, image_parts) = match &augmented {
Some(result) => (result.text.as_str(), result.image_parts.clone()),
None => (content, Vec::new()),
};
// Start the turn and get messages
let turn_messages = {
let mut sess = session.lock().await;
@@ -263,13 +263,12 @@ impl Agent {
.threads
.get_mut(&thread_id)
.ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?;
let turn = thread.start_turn(effective_content);
turn.image_content_parts = image_parts;
thread.start_turn(content);
thread.messages()
};
// Persist user message to DB immediately so it survives crashes
self.persist_user_message(thread_id, &message.user_id, effective_content)
self.persist_user_message(thread_id, &message.user_id, content)
.await;
// Send thinking status
@@ -331,10 +330,10 @@ impl Agent {
};
thread.complete_turn(&response);
let (turn_number, tool_calls) = thread
let tool_calls = thread
.turns
.last()
.map(|t| (t.turn_number, t.tool_calls.clone()))
.map(|t| t.tool_calls.clone())
.unwrap_or_default();
let _ = self
.channels
@@ -346,7 +345,7 @@ impl Agent {
.await;
// Persist tool calls then assistant response (user message already persisted at turn start)
self.persist_tool_calls(thread_id, &message.user_id, turn_number, &tool_calls)
self.persist_tool_calls(thread_id, &message.user_id, &tool_calls)
.await;
self.persist_assistant_response(thread_id, &message.user_id, &response)
.await;
@@ -358,7 +357,7 @@ impl Agent {
let request_id = pending.request_id;
let tool_name = pending.tool_name.clone();
let description = pending.description.clone();
let parameters = pending.display_parameters.clone();
let parameters = pending.parameters.clone();
thread.await_approval(pending);
let _ = self
.channels
@@ -455,7 +454,6 @@ impl Agent {
&self,
thread_id: Uuid,
user_id: &str,
turn_number: usize,
tool_calls: &[crate::agent::session::TurnToolCall],
) {
if tool_calls.is_empty() {
@@ -469,24 +467,14 @@ impl Agent {
let summaries: Vec<serde_json::Value> = tool_calls
.iter()
.enumerate()
.map(|(i, tc)| {
let mut obj = serde_json::json!({
"name": tc.name,
"call_id": format!("turn{}_{}", turn_number, i),
});
.map(|tc| {
let mut obj = serde_json::json!({ "name": tc.name });
if let Some(ref result) = tc.result {
let preview = match result {
serde_json::Value::String(s) => truncate_preview(s, 500),
other => truncate_preview(&other.to_string(), 500),
};
obj["result_preview"] = serde_json::Value::String(preview);
// Store full result (truncated to ~1000 chars) for LLM context rebuild
let full_result = match result {
serde_json::Value::String(s) => truncate_preview(s, 1000),
other => truncate_preview(&other.to_string(), 1000),
};
obj["result"] = serde_json::Value::String(full_result);
}
if let Some(ref error) = tc.error {
obj["error"] = serde_json::Value::String(truncate_preview(error, 200));
@@ -629,7 +617,7 @@ impl Agent {
crate::agent::context_monitor::CompactionStrategy::Summarize { keep_recent: 5 },
);
let compactor = ContextCompactor::new(self.llm().clone());
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
match compactor
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
.await
@@ -745,19 +733,8 @@ impl Agent {
}
// Execute the approved tool and continue the loop
let mut job_ctx =
let job_ctx =
JobContext::with_user(&message.user_id, "chat", "Interactive chat session");
job_ctx.http_interceptor = self.deps.http_interceptor.clone();
// Prefer a valid timezone from the approval message, fall back to the
// resolved timezone stored when the approval was originally requested.
let tz_candidate = message
.timezone
.as_deref()
.filter(|tz| crate::timezone::parse_timezone(tz).is_some())
.or(pending.user_timezone.as_deref());
if let Some(tz) = tz_candidate {
job_ctx.user_timezone = tz.to_string();
}
let _ = self
.channels
@@ -774,17 +751,14 @@ impl Agent {
.execute_chat_tool(&pending.tool_name, &pending.parameters, &job_ctx)
.await;
let tool_ref = self.tools().get(&pending.tool_name).await;
let _ = self
.channels
.send_status(
&message.channel,
StatusUpdate::tool_completed(
pending.tool_name.clone(),
&tool_result,
&pending.display_parameters,
tool_ref.as_deref(),
),
StatusUpdate::ToolCompleted {
name: pending.tool_name.clone(),
success: tool_result.is_ok(),
},
&message.metadata,
)
.await;
@@ -809,33 +783,19 @@ impl Agent {
let mut context_messages = pending.context_messages;
let deferred_tool_calls = pending.deferred_tool_calls;
// Sanitize tool result, then record the cleaned version in the
// thread. Must happen before auth intercept check which may return early.
let is_tool_error = tool_result.is_err();
let result_content = match &tool_result {
Ok(output) => {
let sanitized = self
.safety()
.sanitize_tool_output(&pending.tool_name, output);
self.safety().wrap_for_llm(
&pending.tool_name,
&sanitized.content,
sanitized.was_modified,
)
}
Err(e) => format!("Error: {}", e),
};
// Record sanitized result in thread
// Record result in thread
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id)
&& let Some(turn) = thread.last_turn_mut()
{
if is_tool_error {
turn.record_tool_error(result_content.clone());
} else {
turn.record_tool_result(serde_json::json!(result_content));
match &tool_result {
Ok(output) => {
turn.record_tool_result(serde_json::json!(output));
}
Err(e) => {
turn.record_tool_error(e.to_string());
}
}
}
}
@@ -857,6 +817,21 @@ impl Agent {
return Ok(SubmissionResult::response(instructions));
}
// Add tool result to context
let result_content = match tool_result {
Ok(output) => {
let sanitized = self
.safety()
.sanitize_tool_output(&pending.tool_name, &output);
self.safety().wrap_for_llm(
&pending.tool_name,
&sanitized.content,
sanitized.was_modified,
)
}
Err(e) => format!("Error: {}", e),
};
context_messages.push(ChatMessage::tool_result(
&pending.tool_call_id,
&pending.tool_name,
@@ -933,17 +908,14 @@ impl Agent {
.execute_chat_tool(&tc.name, &tc.arguments, &job_ctx)
.await;
let deferred_tool = self.tools().get(&tc.name).await;
let _ = self
.channels
.send_status(
&message.channel,
StatusUpdate::tool_completed(
tc.name.clone(),
&result,
&tc.arguments,
deferred_tool.as_deref(),
),
StatusUpdate::ToolCompleted {
name: tc.name.clone(),
success: result.is_ok(),
},
&message.metadata,
)
.await;
@@ -985,16 +957,13 @@ impl Agent {
)
.await;
let par_tool = tools.get(&tc.name).await;
let _ = channels
.send_status(
&channel,
StatusUpdate::tool_completed(
tc.name.clone(),
&result,
&tc.arguments,
par_tool.as_deref(),
),
StatusUpdate::ToolCompleted {
name: tc.name.clone(),
success: result.is_ok(),
},
&metadata,
)
.await;
@@ -1061,31 +1030,15 @@ impl Agent {
.await;
}
// Sanitize first, then record the cleaned version in thread.
// Must happen before auth detection which may set deferred_auth.
let is_deferred_error = deferred_result.is_err();
let deferred_content = match &deferred_result {
Ok(output) => {
let sanitized = self.safety().sanitize_tool_output(&tc.name, output);
self.safety().wrap_for_llm(
&tc.name,
&sanitized.content,
sanitized.was_modified,
)
}
Err(e) => format!("Error: {}", e),
};
// Record sanitized result in thread
// Record in thread
{
let mut sess = session.lock().await;
if let Some(thread) = sess.threads.get_mut(&thread_id)
&& let Some(turn) = thread.last_turn_mut()
{
if is_deferred_error {
turn.record_tool_error(deferred_content.clone());
} else {
turn.record_tool_result(serde_json::json!(deferred_content));
match &deferred_result {
Ok(output) => turn.record_tool_result(serde_json::json!(output)),
Err(e) => turn.record_tool_error(e.to_string()),
}
}
}
@@ -1107,6 +1060,18 @@ impl Agent {
deferred_auth = Some(instructions);
}
let deferred_content = match deferred_result {
Ok(output) => {
let sanitized = self.safety().sanitize_tool_output(&tc.name, &output);
self.safety().wrap_for_llm(
&tc.name,
&sanitized.content,
sanitized.was_modified,
)
}
Err(e) => format!("Error: {}", e),
};
context_messages.push(ChatMessage::tool_result(&tc.id, &tc.name, deferred_content));
}
@@ -1121,19 +1086,16 @@ impl Agent {
request_id: Uuid::new_v4(),
tool_name: tc.name.clone(),
parameters: tc.arguments.clone(),
display_parameters: redact_params(&tc.arguments, tool.sensitive_params()),
description: tool.description().to_string(),
tool_call_id: tc.id.clone(),
context_messages: context_messages.clone(),
deferred_tool_calls: deferred_tool_calls[approval_idx + 1..].to_vec(),
// Carry forward the resolved timezone from the original pending approval
user_timezone: pending.user_timezone.clone(),
};
let request_id = new_pending.request_id;
let tool_name = new_pending.tool_name.clone();
let description = new_pending.description.clone();
let parameters = new_pending.display_parameters.clone();
let parameters = new_pending.parameters.clone();
{
let mut sess = session.lock().await;
@@ -1174,13 +1136,13 @@ impl Agent {
match result {
Ok(AgenticLoopResult::Response(response)) => {
thread.complete_turn(&response);
let (turn_number, tool_calls) = thread
let tool_calls = thread
.turns
.last()
.map(|t| (t.turn_number, t.tool_calls.clone()))
.map(|t| t.tool_calls.clone())
.unwrap_or_default();
// User message already persisted at turn start; save tool calls then assistant response
self.persist_tool_calls(thread_id, &message.user_id, turn_number, &tool_calls)
self.persist_tool_calls(thread_id, &message.user_id, &tool_calls)
.await;
self.persist_assistant_response(thread_id, &message.user_id, &response)
.await;
@@ -1200,7 +1162,7 @@ impl Agent {
let request_id = new_pending.request_id;
let tool_name = new_pending.tool_name.clone();
let description = new_pending.description.clone();
let parameters = new_pending.display_parameters.clone();
let parameters = new_pending.parameters.clone();
thread.await_approval(new_pending);
let _ = self
.channels
@@ -1322,7 +1284,7 @@ impl Agent {
};
match ext_mgr.auth(&pending.extension_name, Some(token)).await {
Ok(result) if result.is_authenticated() => {
Ok(result) if result.status == "authenticated" => {
tracing::info!(
"Extension '{}' authenticated via auth mode",
pending.extension_name
@@ -1391,8 +1353,8 @@ impl Agent {
}
}
let msg = result
.instructions()
.map(String::from)
.instructions
.clone()
.unwrap_or_else(|| "Invalid token. Please try again.".to_string());
// Re-emit AuthRequired so web UI re-shows the card
let _ = self
@@ -1402,8 +1364,8 @@ impl Agent {
StatusUpdate::AuthRequired {
extension_name: pending.extension_name.clone(),
instructions: Some(msg.clone()),
auth_url: result.auth_url().map(String::from),
setup_url: result.setup_url().map(String::from),
auth_url: result.auth_url,
setup_url: result.setup_url,
},
&message.metadata,
)
@@ -1495,231 +1457,3 @@ impl Agent {
}
}
}
/// Rebuild full LLM-compatible `ChatMessage` sequence from DB messages.
///
/// Parses `role="tool_calls"` rows to reconstruct `assistant_with_tool_calls`
/// and `tool_result` messages so that the LLM sees the complete tool execution
/// history on thread hydration. Falls back gracefully for legacy rows that
/// lack the enriched fields (`call_id`, `parameters`, `result`).
fn rebuild_chat_messages_from_db(
db_messages: &[crate::history::ConversationMessage],
) -> Vec<ChatMessage> {
let mut result = Vec::new();
for msg in db_messages {
match msg.role.as_str() {
"user" => result.push(ChatMessage::user(&msg.content)),
"assistant" => result.push(ChatMessage::assistant(&msg.content)),
"tool_calls" => {
// Try to parse the enriched JSON and rebuild tool messages.
if let Ok(calls) = serde_json::from_str::<Vec<serde_json::Value>>(&msg.content) {
if calls.is_empty() {
continue;
}
// Check if this is an enriched row (has call_id) or legacy
let has_call_id = calls
.first()
.and_then(|c| c.get("call_id"))
.and_then(|v| v.as_str())
.is_some();
if has_call_id {
// Build assistant_with_tool_calls + tool_result messages
let tool_calls: Vec<ToolCall> = calls
.iter()
.map(|c| ToolCall {
id: c["call_id"].as_str().unwrap_or("call_0").to_string(),
name: c["name"].as_str().unwrap_or("unknown").to_string(),
arguments: c
.get("parameters")
.cloned()
.unwrap_or(serde_json::json!({})),
})
.collect();
// The assistant text for tool_calls is always None here;
// the final assistant response comes as a separate
// "assistant" row after this tool_calls row.
result.push(ChatMessage::assistant_with_tool_calls(None, tool_calls));
// Emit tool_result messages for each call
for c in &calls {
let call_id = c["call_id"].as_str().unwrap_or("call_0").to_string();
let name = c["name"].as_str().unwrap_or("unknown").to_string();
let content = if let Some(err) = c.get("error").and_then(|v| v.as_str())
{
format!("Error: {}", err)
} else if let Some(res) = c.get("result").and_then(|v| v.as_str()) {
res.to_string()
} else if let Some(preview) =
c.get("result_preview").and_then(|v| v.as_str())
{
preview.to_string()
} else {
"OK".to_string()
};
result.push(ChatMessage::tool_result(call_id, name, content));
}
}
// Legacy rows without call_id: skip (will appear as
// simple user/assistant pairs, same as before this fix).
}
}
_ => {} // Skip unknown roles
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rebuild_chat_messages_user_assistant_only() {
let messages = vec![
make_db_msg("user", "Hello"),
make_db_msg("assistant", "Hi there!"),
];
let result = rebuild_chat_messages_from_db(&messages);
assert_eq!(result.len(), 2);
assert_eq!(result[0].role, crate::llm::Role::User);
assert_eq!(result[1].role, crate::llm::Role::Assistant);
}
#[test]
fn test_rebuild_chat_messages_with_enriched_tool_calls() {
let tool_json = serde_json::json!([
{
"name": "memory_search",
"call_id": "call_0",
"parameters": {"query": "test"},
"result": "Found 3 results",
"result_preview": "Found 3 re..."
},
{
"name": "echo",
"call_id": "call_1",
"parameters": {"message": "hi"},
"error": "timeout"
}
]);
let messages = vec![
make_db_msg("user", "Search for test"),
make_db_msg("tool_calls", &tool_json.to_string()),
make_db_msg("assistant", "I found some results."),
];
let result = rebuild_chat_messages_from_db(&messages);
// user + assistant_with_tool_calls + tool_result*2 + assistant
assert_eq!(result.len(), 5);
// user
assert_eq!(result[0].role, crate::llm::Role::User);
// assistant with tool_calls
assert_eq!(result[1].role, crate::llm::Role::Assistant);
assert!(result[1].tool_calls.is_some());
let tcs = result[1].tool_calls.as_ref().unwrap();
assert_eq!(tcs.len(), 2);
assert_eq!(tcs[0].name, "memory_search");
assert_eq!(tcs[0].id, "call_0");
assert_eq!(tcs[1].name, "echo");
// tool results
assert_eq!(result[2].role, crate::llm::Role::Tool);
assert_eq!(result[2].tool_call_id, Some("call_0".to_string()));
assert!(result[2].content.contains("Found 3 results"));
assert_eq!(result[3].role, crate::llm::Role::Tool);
assert_eq!(result[3].tool_call_id, Some("call_1".to_string()));
assert!(result[3].content.contains("Error: timeout"));
// final assistant
assert_eq!(result[4].role, crate::llm::Role::Assistant);
assert_eq!(result[4].content, "I found some results.");
}
#[test]
fn test_rebuild_chat_messages_legacy_tool_calls_skipped() {
// Legacy format: no call_id field
let tool_json = serde_json::json!([
{"name": "echo", "result_preview": "hello"}
]);
let messages = vec![
make_db_msg("user", "Hi"),
make_db_msg("tool_calls", &tool_json.to_string()),
make_db_msg("assistant", "Done"),
];
let result = rebuild_chat_messages_from_db(&messages);
// Legacy rows are skipped, only user + assistant
assert_eq!(result.len(), 2);
assert_eq!(result[0].role, crate::llm::Role::User);
assert_eq!(result[1].role, crate::llm::Role::Assistant);
}
#[test]
fn test_rebuild_chat_messages_empty() {
let result = rebuild_chat_messages_from_db(&[]);
assert!(result.is_empty());
}
#[test]
fn test_rebuild_chat_messages_malformed_tool_calls_json() {
let messages = vec![
make_db_msg("user", "Hi"),
make_db_msg("tool_calls", "not valid json"),
make_db_msg("assistant", "Done"),
];
let result = rebuild_chat_messages_from_db(&messages);
// Malformed JSON is silently skipped
assert_eq!(result.len(), 2);
}
#[test]
fn test_rebuild_chat_messages_multi_turn_with_tools() {
let tool_json_1 = serde_json::json!([
{"name": "search", "call_id": "call_0", "parameters": {}, "result": "found it"}
]);
let tool_json_2 = serde_json::json!([
{"name": "write", "call_id": "call_0", "parameters": {"path": "a.txt"}, "result": "ok"}
]);
let messages = vec![
make_db_msg("user", "Find X"),
make_db_msg("tool_calls", &tool_json_1.to_string()),
make_db_msg("assistant", "Found X"),
make_db_msg("user", "Write it"),
make_db_msg("tool_calls", &tool_json_2.to_string()),
make_db_msg("assistant", "Written"),
];
let result = rebuild_chat_messages_from_db(&messages);
// Turn 1: user + assistant_with_calls + tool_result + assistant = 4
// Turn 2: user + assistant_with_calls + tool_result + assistant = 4
assert_eq!(result.len(), 8);
// Verify turn boundaries
assert_eq!(result[0].content, "Find X");
assert!(result[1].tool_calls.is_some());
assert_eq!(result[2].role, crate::llm::Role::Tool);
assert_eq!(result[3].content, "Found X");
assert_eq!(result[4].content, "Write it");
assert!(result[5].tool_calls.is_some());
assert_eq!(result[6].role, crate::llm::Role::Tool);
assert_eq!(result[7].content, "Written");
}
fn make_db_msg(role: &str, content: &str) -> crate::history::ConversationMessage {
crate::history::ConversationMessage {
id: uuid::Uuid::new_v4(),
role: role.to_string(),
content: content.to_string(),
created_at: chrono::Utc::now(),
}
}
}
+68 -531
View File
@@ -9,18 +9,16 @@ use uuid::Uuid;
use crate::agent::scheduler::WorkerMessage;
use crate::agent::task::TaskOutput;
use crate::channels::web::types::SseEvent;
use crate::context::{ContextManager, JobState};
use crate::db::Database;
use crate::error::Error;
use crate::hooks::HookRegistry;
use crate::llm::{
ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolCall,
ToolSelection,
ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection,
};
use crate::safety::SafetyLayer;
use crate::tools::ToolRegistry;
use crate::tools::rate_limiter::RateLimitResult;
use crate::tools::{ApprovalContext, ToolRegistry, redact_params};
/// Shared dependencies for worker execution.
///
@@ -36,14 +34,6 @@ pub struct WorkerDeps {
pub hooks: Arc<HookRegistry>,
pub timeout: Duration,
pub use_planning: bool,
/// SSE broadcast sender for live job event streaming to the web gateway.
pub sse_tx: Option<tokio::sync::broadcast::Sender<SseEvent>>,
/// Approval context for tool execution. When `None`, all non-`Never` tools are
/// blocked (legacy behavior). When `Some`, the context determines which tools
/// are pre-approved for autonomous execution.
pub approval_context: Option<ApprovalContext>,
/// HTTP interceptor for trace recording/replay (propagated to JobContext).
pub http_interceptor: Option<Arc<dyn crate::llm::recording::HttpInterceptor>>,
}
/// Worker that executes a single job.
@@ -108,90 +98,18 @@ impl Worker {
}
}
/// Fire-and-forget persistence of a job event and SSE broadcast.
/// Fire-and-forget persistence of a job event.
fn log_event(&self, event_type: &str, data: serde_json::Value) {
let job_id = self.job_id;
// Persist to DB
if let Some(store) = self.store() {
let store = store.clone();
let et = event_type.to_string();
let d = data.clone();
let job_id = self.job_id;
let event_type = event_type.to_string();
tokio::spawn(async move {
if let Err(e) = store.save_job_event(job_id, &et, &d).await {
if let Err(e) = store.save_job_event(job_id, &event_type, &data).await {
tracing::warn!("Failed to persist event for job {}: {}", job_id, e);
}
});
}
// Broadcast SSE for live web UI updates
if let Some(ref tx) = self.deps.sse_tx {
let job_id_str = job_id.to_string();
let event = match event_type {
"message" => Some(SseEvent::JobMessage {
job_id: job_id_str,
role: data
.get("role")
.and_then(|v| v.as_str())
.unwrap_or("assistant")
.to_string(),
content: data
.get("content")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
}),
"tool_use" => Some(SseEvent::JobToolUse {
job_id: job_id_str,
tool_name: data
.get("tool_name")
.and_then(|v| v.as_str())
.unwrap_or("unknown")
.to_string(),
input: data
.get("input")
.cloned()
.unwrap_or(serde_json::Value::Null),
}),
"tool_result" => Some(SseEvent::JobToolResult {
job_id: job_id_str,
tool_name: data
.get("tool_name")
.and_then(|v| v.as_str())
.unwrap_or("unknown")
.to_string(),
output: data
.get("output")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
}),
"status" => Some(SseEvent::JobStatus {
job_id: job_id_str,
message: data
.get("message")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string(),
}),
"result" => Some(SseEvent::JobResult {
job_id: job_id_str,
status: data
.get("status")
.and_then(|v| v.as_str())
.unwrap_or("completed")
.to_string(),
session_id: data
.get("session_id")
.and_then(|v| v.as_str())
.map(|s| s.to_string()),
}),
_ => None,
};
if let Some(event) = event {
let _ = tx.send(event);
}
}
}
/// Run the worker until the job is complete or stopped.
@@ -205,14 +123,14 @@ impl Worker {
tracing::debug!("Worker for job {} stopped before starting", self.job_id);
return Ok(());
}
Some(WorkerMessage::Ping) | Some(WorkerMessage::UserMessage(_)) => {}
Some(WorkerMessage::Ping) => {}
}
// Get job context
let job_ctx = self.context_manager().get_context(self.job_id).await?;
// Create reasoning engine
let reasoning = Reasoning::new(self.llm().clone());
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
// Build initial reasoning context (tool definitions refreshed each iteration in execution_loop)
let mut reason_ctx = ReasoningContext::new().with_job(&job_ctx.description);
@@ -253,9 +171,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
// Already in a terminal state (e.g. execution_loop
// called mark_completed itself).
}
Ok(JobState::Completed) => {
// execution_loop already called mark_completed.
}
Ok(JobState::Stuck) => {
// execution_loop marked this as stuck (e.g. "plan
// completed but work remains"); leave for self-repair.
@@ -304,10 +219,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
.unwrap_or(50) as usize;
let max_iterations = max_iterations.min(MAX_WORKER_ITERATIONS);
let mut iteration = 0;
const MAX_CONSECUTIVE_RATE_LIMITS: usize = 10;
let mut consecutive_rate_limits = 0usize;
const MAX_TOOL_INTENT_NUDGES: u32 = 2;
let mut consecutive_tool_intent_nudges: u32 = 0;
// Initial tool definitions for planning (will be refreshed in loop)
reason_ctx.available_tools = self.tools().tool_definitions().await;
@@ -358,29 +269,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
None
};
// If we have a plan, execute it. Two exit paths:
// 1. Plan ran to completion → job is Completed or needs continuation
// (check state and only fall through if not terminal)
// 2. Plan was interrupted by UserMessage → fall through to direct loop
// If we have a plan, execute it
if let Some(ref plan) = plan {
self.execute_plan(rx, reasoning, reason_ctx, plan).await?;
// If the plan marked the job completed, terminal, or stuck, we're
// done. Only fall through to the direct selection loop if the
// plan was interrupted or explicitly left the job in-progress.
if let Ok(ctx) = self.context_manager().get_context(self.job_id).await
&& (ctx.state.is_terminal()
|| ctx.state == JobState::Stuck
|| ctx.state == JobState::Completed)
{
return Ok(());
}
return self.execute_plan(rx, reasoning, reason_ctx, plan).await;
}
// Direct tool selection loop (also used as fallback after plan interruption)
// Otherwise, use direct tool selection loop
loop {
// Check for stop signal and injected user messages
while let Ok(msg) = rx.try_recv() {
// Check for stop signal
if let Ok(msg) = rx.try_recv() {
match msg {
WorkerMessage::Stop => {
tracing::debug!("Worker for job {} received stop signal", self.job_id);
@@ -390,20 +287,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
tracing::trace!("Worker for job {} received ping", self.job_id);
}
WorkerMessage::Start => {}
WorkerMessage::UserMessage(content) => {
tracing::info!(
job_id = %self.job_id,
"Worker received follow-up user message"
);
reason_ctx.messages.push(ChatMessage::user(&content));
self.log_event(
"message",
serde_json::json!({
"role": "user",
"content": content,
}),
);
}
}
}
@@ -424,64 +307,12 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
// Refresh tool definitions so newly built tools become visible
reason_ctx.available_tools = self.tools().tool_definitions().await;
// Select next tool(s) to use, with rate-limit retry.
let selections = match reasoning.select_tools(reason_ctx).await {
Ok(s) => s,
Err(crate::error::LlmError::RateLimited { retry_after, .. }) => {
consecutive_rate_limits += 1;
let wait = retry_after.unwrap_or(Duration::from_secs(5));
tracing::warn!(
job_id = %self.job_id,
wait_secs = wait.as_secs(),
attempt = consecutive_rate_limits,
"LLM rate limited during tool selection, backing off"
);
if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS {
self.mark_stuck("Persistent rate limiting").await?;
return Ok(());
}
self.log_event(
"status",
serde_json::json!({
"message": format!("Rate limited, retrying in {}s ({}/{})...",
wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS),
}),
);
tokio::time::sleep(wait).await;
continue;
}
Err(e) => return Err(e.into()),
};
// Select next tool(s) to use
let selections = reasoning.select_tools(reason_ctx).await?;
if selections.is_empty() {
// No tools from select_tools, ask LLM directly (may still return tool calls)
let respond_output = match reasoning.respond_with_tools(reason_ctx).await {
Ok(o) => o,
Err(crate::error::LlmError::RateLimited { retry_after, .. }) => {
consecutive_rate_limits += 1;
let wait = retry_after.unwrap_or(Duration::from_secs(5));
tracing::warn!(
job_id = %self.job_id,
wait_secs = wait.as_secs(),
attempt = consecutive_rate_limits,
"LLM rate limited during respond_with_tools, backing off"
);
if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS {
self.mark_stuck("Persistent rate limiting").await?;
return Ok(());
}
self.log_event(
"status",
serde_json::json!({
"message": format!("Rate limited, retrying in {}s ({}/{})...",
wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS),
}),
);
tokio::time::sleep(wait).await;
continue;
}
Err(e) => return Err(e.into()),
};
let respond_output = reasoning.respond_with_tools(reason_ctx).await?;
match respond_output.result {
RespondResult::Text(response) => {
@@ -505,34 +336,17 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
}),
);
// Nudge the LLM if it expressed tool intent without calling tools
let signals_intent = !reason_ctx.available_tools.is_empty()
&& crate::llm::llm_signals_tool_intent(&response);
if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES
{
consecutive_tool_intent_nudges += 1;
tracing::info!(
job_id = %self.job_id,
"LLM expressed tool intent without calling a tool, nudging"
);
reason_ctx
.messages
.push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE));
} else if !signals_intent {
consecutive_tool_intent_nudges = 0;
if iteration > 3 && iteration % 5 == 0 {
// Generic fallback nudge
reason_ctx.messages.push(ChatMessage::user(
"Are you stuck? Do you need help completing this job?",
));
}
// Give it one more chance to select a tool
if iteration > 3 && iteration % 5 == 0 {
reason_ctx.messages.push(ChatMessage::user(
"Are you stuck? Do you need help completing this job?",
));
}
}
RespondResult::ToolCalls {
tool_calls,
content,
} => {
consecutive_tool_intent_nudges = 0;
// Model returned tool calls - execute them
tracing::debug!(
"Job {} respond_with_tools returned {} tool calls",
@@ -577,62 +391,39 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
}
}
}
} else if selections.len() == 1 {
// Single tool: execute directly
let selection = &selections[0];
tracing::debug!(
"Job {} selecting tool: {} - {}",
self.job_id,
selection.tool_name,
selection.reasoning
);
let result = self
.execute_tool(&selection.tool_name, &selection.parameters)
.await;
self.process_tool_result(reason_ctx, selection, result)
.await?;
} else {
consecutive_tool_intent_nudges = 0;
// Multiple tools: execute in parallel
tracing::debug!(
"Job {} executing {} tools in parallel",
self.job_id,
selections.len()
);
// Record the assistant tool_calls message so that tool_result
// messages have a matching parent (prevents orphaned rewrites).
let tool_calls: Vec<ToolCall> = selections
.iter()
.map(|s| ToolCall {
id: s.tool_call_id.clone(),
name: s.tool_name.clone(),
arguments: s.parameters.clone(),
})
.collect();
reason_ctx
.messages
.push(ChatMessage::assistant_with_tool_calls(None, tool_calls));
let results = self.execute_tools_parallel(&selections).await;
if selections.len() == 1 {
// Single tool: execute directly
let selection = &selections[0];
tracing::debug!(
"Job {} selecting tool: {} - {}",
self.job_id,
selection.tool_name,
selection.reasoning
);
let result = self
.execute_tool(&selection.tool_name, &selection.parameters)
.await;
self.process_tool_result(reason_ctx, selection, result)
// Process all results
for (selection, result) in selections.iter().zip(results) {
self.process_tool_result(reason_ctx, selection, result.result)
.await?;
} else {
// Multiple tools: execute in parallel
tracing::debug!(
"Job {} executing {} tools in parallel",
self.job_id,
selections.len()
);
let results = self.execute_tools_parallel(&selections).await;
// Process all results
for (selection, result) in selections.iter().zip(results) {
self.process_tool_result(reason_ctx, selection, result.result)
.await?;
}
}
}
// Reset rate-limit counter after a successful iteration (all LLM
// calls succeeded). Placed here so alternating success/fail between
// select_tools and respond_with_tools cannot bypass the cap.
consecutive_rate_limits = 0;
// Small delay between iterations
tokio::time::sleep(Duration::from_millis(100)).await;
}
@@ -720,11 +511,8 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
name: tool_name.to_string(),
})?;
// Check approval: use context-aware check if available, else block all non-Never tools
let requirement = tool.requires_approval(params);
let blocked =
ApprovalContext::is_blocked_or_default(&deps.approval_context, tool_name, requirement);
if blocked {
// Tools requiring approval are blocked in autonomous jobs
if tool.requires_approval(params).is_required() {
return Err(crate::error::ToolError::AuthRequired {
name: tool_name.to_string(),
}
@@ -732,11 +520,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
}
// Fetch job context early so we have the real user_id for hooks and rate limiting
let mut job_ctx = deps.context_manager.get_context(job_id).await?;
// Propagate http_interceptor for trace recording/replay
if job_ctx.http_interceptor.is_none() {
job_ctx.http_interceptor = deps.http_interceptor.clone();
}
let job_ctx = deps.context_manager.get_context(job_id).await?;
// Check per-tool rate limit before running hooks or executing (cheaper check first)
if let Some(config) = tool.rate_limit_config()
@@ -756,10 +540,9 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
// Run BeforeToolCall hook
let params = {
use crate::hooks::{HookError, HookEvent, HookOutcome};
let hook_params = redact_params(params, tool.sensitive_params());
let event = HookEvent::ToolCall {
tool_name: tool_name.to_string(),
parameters: hook_params,
parameters: params.clone(),
user_id: job_ctx.user_id.clone(),
context: format!("job:{}", job_id),
};
@@ -815,12 +598,9 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
.into());
}
// Redact sensitive parameter values (e.g. secret_save's "value") before
// they touch any observability or audit path.
let safe_params = redact_params(&params, tool.sensitive_params());
tracing::debug!(
tool = %tool_name,
params = %safe_params,
params = %params,
job = %job_id,
"Tool call started"
);
@@ -872,7 +652,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
match deps
.context_manager
.update_memory(job_id, |mem| {
let rec = mem.create_action(tool_name, safe_params.clone()).succeed(
let rec = mem.create_action(tool_name, params.clone()).succeed(
output_str.clone(),
output.result.clone(),
elapsed,
@@ -894,7 +674,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
.context_manager
.update_memory(job_id, |mem| {
let rec = mem
.create_action(tool_name, safe_params.clone())
.create_action(tool_name, params.clone())
.fail(e.to_string(), elapsed);
mem.record_action(rec.clone());
rec
@@ -913,7 +693,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
.context_manager
.update_memory(job_id, |mem| {
let rec = mem
.create_action(tool_name, safe_params.clone())
.create_action(tool_name, params.clone())
.fail("Execution timeout", elapsed);
mem.record_action(rec.clone());
rec
@@ -1056,8 +836,8 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
plan: &ActionPlan,
) -> Result<(), Error> {
for (i, action) in plan.actions.iter().enumerate() {
// Check for stop signal and injected user messages
while let Ok(msg) = rx.try_recv() {
// Check for stop signal
if let Ok(msg) = rx.try_recv() {
match msg {
WorkerMessage::Stop => {
tracing::debug!(
@@ -1070,29 +850,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
tracing::trace!("Worker for job {} received ping", self.job_id);
}
WorkerMessage::Start => {}
WorkerMessage::UserMessage(content) => {
tracing::info!(
job_id = %self.job_id,
"User message received during plan execution, abandoning plan"
);
reason_ctx.messages.push(ChatMessage::user(&content));
self.log_event(
"message",
serde_json::json!({
"role": "user",
"content": content,
}),
);
self.log_event(
"status",
serde_json::json!({
"message": "Plan interrupted by user message, re-evaluating...",
}),
);
// Return Ok to break out of plan; caller falls through to
// the direct selection loop for LLM re-evaluation.
return Ok(());
}
}
}
@@ -1105,6 +862,11 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
action.reasoning
);
// Execute the planned tool
let result = self
.execute_tool(&action.tool_name, &action.parameters)
.await;
// Create a synthetic ToolSelection for process_tool_result.
// Plan actions don't originate from an LLM tool_call response so
// there is no real tool_call_id; generate a unique one.
@@ -1116,24 +878,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
tool_call_id: format!("plan_{}_{}", self.job_id, i),
};
// Record the assistant tool_calls message so that the tool_result
// has a matching parent (prevents orphaned rewrites).
reason_ctx
.messages
.push(ChatMessage::assistant_with_tool_calls(
None,
vec![ToolCall {
id: selection.tool_call_id.clone(),
name: selection.tool_name.clone(),
arguments: selection.parameters.clone(),
}],
));
// Execute the planned tool
let result = self
.execute_tool(&action.tool_name, &action.parameters)
.await;
// Process the result
let completed = self
.process_tool_result(reason_ctx, &selection, result)
@@ -1158,18 +902,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
if crate::util::llm_signals_completion(&response) {
self.mark_completed().await?;
} else {
// Job not complete — return Ok without marking terminal so the
// caller falls through to the direct selection loop for continuation.
// Job not complete, could re-plan or fall back to direct selection
tracing::info!(
"Job {} plan completed but work remains, falling back to direct selection",
self.job_id
);
self.log_event(
"status",
serde_json::json!({
"message": "Plan completed but job needs more work, continuing...",
}),
);
// Continue with standard execution loop by returning (will be picked up by main loop)
self.mark_stuck("Plan completed but job incomplete - needs re-planning")
.await?;
}
Ok(())
@@ -1200,7 +940,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
self.log_event(
"result",
serde_json::json!({
"status": "completed",
"success": true,
"message": "Job completed successfully",
}),
@@ -1226,7 +965,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
self.log_event(
"result",
serde_json::json!({
"status": "failed",
"success": false,
"message": format!("Execution failed: {}", reason),
}),
@@ -1247,7 +985,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
self.log_event(
"result",
serde_json::json!({
"status": "stuck",
"success": false,
"message": format!("Job stuck: {}", reason),
}),
@@ -1366,9 +1103,6 @@ mod tests {
hooks: Arc::new(crate::hooks::HookRegistry::new()),
timeout: Duration::from_secs(30),
use_planning: false,
sse_tx: None,
approval_context: None,
http_interceptor: None,
};
Worker::new(job_id, deps)
@@ -1485,11 +1219,9 @@ mod tests {
assert!(r.result.is_ok(), "Tool should succeed");
}
// Parallel should complete well under the sequential 600ms threshold.
// Use a generous bound (800ms) to avoid flaky failures on slow CI runners,
// while still proving parallelism (sequential would be >= 600ms on any machine).
assert!(
elapsed < Duration::from_millis(800),
"Parallel execution took {:?}, expected < 800ms (sequential would be ~600ms)",
elapsed < Duration::from_millis(500),
"Parallel execution took {:?}, expected < 500ms",
elapsed
);
}
@@ -1567,199 +1299,4 @@ mod tests {
"Missing tool should produce an error, not a panic"
);
}
/// Verify that calling mark_completed on an already-Completed job returns
/// an error (Completed → Completed is an invalid state transition).
#[tokio::test]
async fn test_mark_completed_twice_returns_error() {
let worker = make_worker(vec![]).await;
// Transition to InProgress first (required by state machine)
worker
.context_manager()
.update_context(worker.job_id, |ctx| {
ctx.transition_to(JobState::InProgress, None)
})
.await
.unwrap()
.unwrap();
// First mark_completed should succeed
worker.mark_completed().await.unwrap();
// Verify state is Completed
let ctx = worker
.context_manager()
.get_context(worker.job_id)
.await
.unwrap();
assert_eq!(ctx.state, JobState::Completed);
// Second mark_completed should fail (Completed → Completed is invalid)
let result = worker.mark_completed().await;
assert!(
result.is_err(),
"Completed → Completed transition should be rejected by state machine"
);
}
/// Build a Worker with the given approval context.
async fn make_worker_with_approval(
tools: Vec<Arc<dyn Tool>>,
approval_context: Option<crate::tools::ApprovalContext>,
) -> Worker {
let registry = ToolRegistry::new();
for t in tools {
registry.register(t).await;
}
let cm = Arc::new(crate::context::ContextManager::new(5));
let job_id = cm.create_job("test", "test job").await.unwrap();
let deps = WorkerDeps {
context_manager: cm,
llm: Arc::new(StubLlm),
safety: Arc::new(SafetyLayer::new(&SafetyConfig {
max_output_length: 100_000,
injection_check_enabled: false,
})),
tools: Arc::new(registry),
store: None,
hooks: Arc::new(crate::hooks::HookRegistry::new()),
timeout: Duration::from_secs(30),
use_planning: false,
sse_tx: None,
approval_context,
http_interceptor: None,
};
Worker::new(job_id, deps)
}
/// A tool that requires approval (UnlessAutoApproved).
struct ApprovalTool;
#[async_trait::async_trait]
impl Tool for ApprovalTool {
fn name(&self) -> &str {
"needs_approval"
}
fn description(&self) -> &str {
"Tool requiring approval"
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({"type": "object", "properties": {}})
}
async fn execute(
&self,
_params: serde_json::Value,
_ctx: &crate::context::JobContext,
) -> Result<ToolOutput, crate::tools::ToolError> {
Ok(ToolOutput::text(
"approved",
std::time::Instant::now().elapsed(),
))
}
fn requires_approval(
&self,
_params: &serde_json::Value,
) -> crate::tools::ApprovalRequirement {
crate::tools::ApprovalRequirement::UnlessAutoApproved
}
fn requires_sanitization(&self) -> bool {
false
}
}
/// A tool that always requires approval.
struct AlwaysApprovalTool;
#[async_trait::async_trait]
impl Tool for AlwaysApprovalTool {
fn name(&self) -> &str {
"always_approval"
}
fn description(&self) -> &str {
"Tool always requiring approval"
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({"type": "object", "properties": {}})
}
async fn execute(
&self,
_params: serde_json::Value,
_ctx: &crate::context::JobContext,
) -> Result<ToolOutput, crate::tools::ToolError> {
Ok(ToolOutput::text(
"always",
std::time::Instant::now().elapsed(),
))
}
fn requires_approval(
&self,
_params: &serde_json::Value,
) -> crate::tools::ApprovalRequirement {
crate::tools::ApprovalRequirement::Always
}
fn requires_sanitization(&self) -> bool {
false
}
}
#[tokio::test]
async fn test_approval_context_unblocks_unless_auto_approved() {
// Without approval context, UnlessAutoApproved is blocked
let worker_blocked = make_worker_with_approval(vec![Arc::new(ApprovalTool)], None).await;
let result = worker_blocked
.execute_tool("needs_approval", &serde_json::json!({}))
.await;
assert!(
result.is_err(),
"Should be blocked without approval context"
);
// With autonomous approval context, UnlessAutoApproved is allowed
let worker_allowed = make_worker_with_approval(
vec![Arc::new(ApprovalTool)],
Some(crate::tools::ApprovalContext::autonomous()),
)
.await;
let result = worker_allowed
.execute_tool("needs_approval", &serde_json::json!({}))
.await;
assert!(result.is_ok(), "Should be allowed with autonomous context");
}
#[tokio::test]
async fn test_approval_context_blocks_always_unless_permitted() {
// Autonomous context without tool_permissions blocks Always tools
let worker_blocked = make_worker_with_approval(
vec![Arc::new(AlwaysApprovalTool)],
Some(crate::tools::ApprovalContext::autonomous()),
)
.await;
let result = worker_blocked
.execute_tool("always_approval", &serde_json::json!({}))
.await;
assert!(
result.is_err(),
"Always tool should be blocked without permission"
);
// Autonomous context with tool_permissions allows Always tools
let worker_allowed = make_worker_with_approval(
vec![Arc::new(AlwaysApprovalTool)],
Some(crate::tools::ApprovalContext::autonomous_with_tools([
"always_approval".to_string(),
])),
)
.await;
let result = worker_allowed
.execute_tool("always_approval", &serde_json::json!({}))
.await;
assert!(
result.is_ok(),
"Always tool should be allowed with permission"
);
}
}
+113 -313
View File
@@ -15,13 +15,13 @@ use crate::context::ContextManager;
use crate::db::Database;
use crate::extensions::ExtensionManager;
use crate::hooks::HookRegistry;
use crate::llm::{LlmProvider, RecordingLlm, SessionManager};
use crate::llm::{LlmProvider, SessionManager};
use crate::safety::SafetyLayer;
use crate::secrets::SecretsStore;
use crate::skills::SkillRegistry;
use crate::skills::catalog::SkillCatalog;
use crate::tools::ToolRegistry;
use crate::tools::mcp::{McpProcessManager, McpSessionManager};
use crate::tools::mcp::McpSessionManager;
use crate::tools::wasm::SharedCredentialRegistry;
use crate::tools::wasm::WasmToolRuntime;
use crate::workspace::{EmbeddingProvider, Workspace};
@@ -41,7 +41,6 @@ pub struct AppComponents {
pub workspace: Option<Arc<Workspace>>,
pub extension_manager: Option<Arc<ExtensionManager>>,
pub mcp_session_manager: Arc<McpSessionManager>,
pub mcp_process_manager: Arc<McpProcessManager>,
pub wasm_tool_runtime: Option<Arc<WasmToolRuntime>>,
pub log_broadcaster: Arc<LogBroadcaster>,
pub context_manager: Arc<ContextManager>,
@@ -49,7 +48,6 @@ pub struct AppComponents {
pub skill_registry: Option<Arc<std::sync::RwLock<SkillRegistry>>>,
pub skill_catalog: Option<Arc<SkillCatalog>>,
pub cost_guard: Arc<crate::agent::cost_guard::CostGuard>,
pub recording_handle: Option<Arc<RecordingLlm>>,
pub session: Arc<SessionManager>,
pub catalog_entries: Vec<crate::extensions::RegistryEntry>,
pub dev_loaded_tool_names: Vec<String>,
@@ -73,9 +71,6 @@ pub struct AppBuilder {
db: Option<Arc<dyn Database>>,
secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>,
// Test overrides
llm_override: Option<Arc<dyn LlmProvider>>,
// Backend-specific handles needed by secrets store
#[cfg(feature = "postgres")]
pg_pool: Option<deadpool_postgres::Pool>,
@@ -104,7 +99,6 @@ impl AppBuilder {
log_broadcaster,
db: None,
secrets_store: None,
llm_override: None,
#[cfg(feature = "postgres")]
pg_pool: None,
#[cfg(feature = "libsql")]
@@ -112,26 +106,11 @@ impl AppBuilder {
}
}
/// Inject a pre-created database, skipping `init_database()`.
pub fn with_database(&mut self, db: Arc<dyn Database>) {
self.db = Some(db);
}
/// Inject a pre-created LLM provider, skipping `init_llm()`.
pub fn with_llm(&mut self, llm: Arc<dyn LlmProvider>) {
self.llm_override = Some(llm);
}
/// Phase 1: Initialize database backend.
///
/// Creates the database connection, runs migrations, reloads config
/// from DB, attaches DB to session manager, and cleans up stale jobs.
pub async fn init_database(&mut self) -> Result<(), anyhow::Error> {
if self.db.is_some() {
tracing::debug!("Database already provided, skipping init_database()");
return Ok(());
}
if self.flags.no_db {
tracing::warn!("Running without database connection");
return Ok(());
@@ -245,31 +224,11 @@ impl AppBuilder {
let master_key = match self.config.secrets.master_key() {
Some(k) => k,
None => {
// No secrets DB available, but we can still load tokens from
// OS credential stores (e.g., Anthropic OAuth via Claude Code's
// macOS Keychain / Linux ~/.claude/.credentials.json).
crate::config::inject_os_credentials();
// Consume unused handles
#[cfg(feature = "libsql")]
{
self.libsql_db.take();
}
// Re-resolve only the LLM config with OS credentials.
let store: Option<&(dyn crate::db::SettingsStore + Sync)> =
self.db.as_ref().map(|db| db.as_ref() as _);
let toml_path = self.toml_path.as_deref();
if let Err(e) = self
.config
.re_resolve_llm(store, "default", toml_path)
.await
{
tracing::warn!(
"Failed to re-resolve LLM config after OS credential injection: {e}"
);
}
return Ok(());
}
};
@@ -312,16 +271,18 @@ impl AppBuilder {
// Inject LLM API keys from encrypted storage
crate::config::inject_llm_keys_from_secrets(secrets.as_ref(), "default").await;
// Re-resolve only the LLM config with newly available keys.
let store: Option<&(dyn crate::db::SettingsStore + Sync)> =
self.db.as_ref().map(|db| db.as_ref() as _);
let toml_path = self.toml_path.as_deref();
if let Err(e) = self
.config
.re_resolve_llm(store, "default", toml_path)
.await
{
tracing::warn!("Failed to re-resolve LLM config after secret injection: {e}");
// Re-resolve config with newly available keys
if let Some(ref db) = self.db {
let toml_path = self.toml_path.as_deref();
match Config::from_db_with_toml(db.as_ref(), "default", toml_path).await {
Ok(refreshed) => {
self.config = refreshed;
tracing::debug!("LlmConfig re-resolved after secret injection");
}
Err(e) => {
tracing::warn!("Failed to re-resolve config after secret injection: {}", e);
}
}
}
}
@@ -334,19 +295,12 @@ impl AppBuilder {
/// Delegates to `build_provider_chain` which applies all decorators
/// (retry, smart routing, failover, circuit breaker, response cache).
#[allow(clippy::type_complexity)]
pub async fn init_llm(
pub fn init_llm(
&self,
) -> Result<
(
Arc<dyn LlmProvider>,
Option<Arc<dyn LlmProvider>>,
Option<Arc<RecordingLlm>>,
),
anyhow::Error,
> {
let (llm, cheap_llm, recording_handle) =
crate::llm::build_provider_chain(&self.config.llm, self.session.clone()).await?;
Ok((llm, cheap_llm, recording_handle))
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), anyhow::Error> {
let (llm, cheap_llm) =
crate::llm::build_provider_chain(&self.config.llm, self.session.clone())?;
Ok((llm, cheap_llm))
}
/// Phase 4: Initialize safety, tools, embeddings, and workspace.
@@ -377,16 +331,27 @@ impl AppBuilder {
};
tools.register_builtin_tools();
if let Some(ref ss) = self.secrets_store {
tools.register_secrets_tools(Arc::clone(ss));
}
// Create embeddings provider using the unified method
let embeddings = self
.config
.embeddings
.create_provider(&self.config.llm.nearai.base_url, self.session.clone());
// Warn if libSQL backend is used with non-1536 embedding dimension.
if self.config.database.backend == crate::config::DatabaseBackend::LibSql
&& self.config.embeddings.enabled
&& self.config.embeddings.dimension != 1536
{
tracing::warn!(
configured_dimension = self.config.embeddings.dimension,
"Embedding dimension {} is not 1536. The libSQL schema uses \
F32_BLOB(1536) which requires exactly 1536 dimensions. \
Embedding storage will fail. Use PostgreSQL or set \
EMBEDDING_DIMENSION=1536.",
self.config.embeddings.dimension
);
}
// Register memory tools if database is available
let workspace = if let Some(ref db) = self.db {
let mut ws = Workspace::new_with_db("default", db.clone());
@@ -400,55 +365,16 @@ impl AppBuilder {
None
};
// Register image/vision tools if we have a workspace and LLM API credentials
if workspace.is_some() {
let (api_base, api_key_opt) = if let Some(ref provider) = self.config.llm.provider {
(
provider.base_url.clone(),
provider.api_key.as_ref().map(|s| {
use secrecy::ExposeSecret;
s.expose_secret().to_string()
}),
)
} else {
(
self.config.llm.nearai.base_url.clone(),
self.config.llm.nearai.api_key.as_ref().map(|s| {
use secrecy::ExposeSecret;
s.expose_secret().to_string()
}),
)
};
if let Some(api_key) = api_key_opt {
// Check for image generation models
let model_name = self
.config
.llm
.provider
.as_ref()
.map(|p| p.model.clone())
.unwrap_or_else(|| self.config.llm.nearai.model.clone());
let models = vec![model_name.clone()];
let gen_model = crate::llm::image_models::suggest_image_model(&models)
.unwrap_or("flux-1.1-pro")
.to_string();
tools.register_image_tools(api_base.clone(), api_key.clone(), gen_model, None);
// Check for vision models
let vision_model = crate::llm::vision_models::suggest_vision_model(&models)
.unwrap_or(&model_name)
.to_string();
tools.register_vision_tools(api_base, api_key, vision_model, None);
}
}
// Register builder tool if enabled
if self.config.builder.enabled
&& (self.config.agent.allow_local_tools || !self.config.sandbox.enabled)
{
tools
.register_builder_tool(llm.clone(), Some(self.config.builder.to_builder_config()))
.register_builder_tool(
llm.clone(),
safety.clone(),
Some(self.config.builder.to_builder_config()),
)
.await;
tracing::info!("Builder mode enabled");
}
@@ -464,7 +390,6 @@ impl AppBuilder {
) -> Result<
(
Arc<McpSessionManager>,
Arc<McpProcessManager>,
Option<Arc<WasmToolRuntime>>,
Option<Arc<ExtensionManager>>,
Vec<crate::extensions::RegistryEntry>,
@@ -472,13 +397,10 @@ impl AppBuilder {
),
anyhow::Error,
> {
use crate::tools::mcp::{
McpClient, McpTransport, config::load_mcp_servers_from_db, is_authenticated,
};
use crate::tools::mcp::{McpClient, config::load_mcp_servers_from_db, is_authenticated};
use crate::tools::wasm::{WasmToolLoader, load_dev_tools};
let mcp_session_manager = Arc::new(McpSessionManager::new());
let mcp_process_manager = Arc::new(McpProcessManager::new());
// Create WASM tool runtime eagerly so extensions installed after startup
// (e.g. via the web UI) can still be activated. The tools directory is only
@@ -554,175 +476,97 @@ impl AppBuilder {
let db = self.db.clone();
let tools = Arc::clone(tools);
let mcp_sm = Arc::clone(&mcp_session_manager);
let pm = Arc::clone(&mcp_process_manager);
async move {
let servers_result = if let Some(ref d) = db {
load_mcp_servers_from_db(d.as_ref(), "default").await
} else {
crate::tools::mcp::config::load_mcp_servers().await
};
match servers_result {
Ok(servers) => {
let enabled: Vec<_> = servers.enabled_servers().cloned().collect();
if !enabled.is_empty() {
tracing::info!("Loading {} configured MCP server(s)...", enabled.len());
}
if let Some(ref secrets) = secrets_store {
let servers_result = if let Some(ref d) = db {
load_mcp_servers_from_db(d.as_ref(), "default").await
} else {
crate::tools::mcp::config::load_mcp_servers().await
};
match servers_result {
Ok(servers) => {
let enabled: Vec<_> = servers.enabled_servers().cloned().collect();
if !enabled.is_empty() {
tracing::info!(
"Loading {} configured MCP server(s)...",
enabled.len()
);
}
let mut join_set = tokio::task::JoinSet::new();
for server in enabled {
let mcp_sm = Arc::clone(&mcp_sm);
let secrets = secrets_store.clone();
let tools = Arc::clone(&tools);
let pm = Arc::clone(&pm);
let mut join_set = tokio::task::JoinSet::new();
for server in enabled {
let mcp_sm = Arc::clone(&mcp_sm);
let secrets = Arc::clone(secrets);
let tools = Arc::clone(&tools);
join_set.spawn(async move {
let server_name = server.name.clone();
join_set.spawn(async move {
let server_name = server.name.clone();
let has_tokens =
is_authenticated(&server, &secrets, "default").await;
let client: McpClient = match server.effective_transport() {
crate::tools::mcp::config::EffectiveTransport::Stdio {
command,
args,
env,
} => {
match pm
.spawn_stdio(
&server_name,
command,
args.to_vec(),
env.clone(),
)
.await
{
Ok(transport) => McpClient::new_with_transport(
&server_name,
transport as Arc<dyn McpTransport>,
None,
secrets,
"default",
Some(server),
),
Err(e) => {
tracing::warn!(
"Failed to spawn stdio MCP server '{}': {}",
server_name,
e
);
return;
}
}
}
#[cfg(unix)]
crate::tools::mcp::config::EffectiveTransport::Unix {
socket_path,
} => {
match crate::tools::mcp::unix_transport::UnixMcpTransport::connect(
&server_name,
socket_path,
let client = if has_tokens || server.requires_auth() {
McpClient::new_authenticated(
server, mcp_sm, secrets, "default",
)
.await
{
Ok(transport) => McpClient::new_with_transport(
&server_name,
Arc::new(transport) as Arc<dyn McpTransport>,
None,
secrets,
"default",
Some(server),
),
Err(e) => {
tracing::warn!(
"Failed to connect to Unix MCP server '{}': {}",
server_name,
e
);
return;
}
}
}
#[cfg(not(unix))]
crate::tools::mcp::config::EffectiveTransport::Unix { .. } => {
tracing::warn!(
"Unix socket transport is not supported on this platform (server '{}')",
server_name
);
return;
}
crate::tools::mcp::config::EffectiveTransport::Http => {
if let Some(ref secrets) = secrets {
let has_tokens =
is_authenticated(&server, secrets, "default")
.await;
} else {
McpClient::new_with_name(&server_name, &server.url)
};
if has_tokens || server.requires_auth() {
McpClient::new_authenticated(
server,
Arc::clone(&mcp_sm),
Arc::clone(secrets),
"default",
)
} else {
McpClient::new_with_config(server)
}
} else {
McpClient::new_with_config(server)
}
}
};
match client.list_tools().await {
Ok(mcp_tools) => {
let tool_count = mcp_tools.len();
match client.create_tools().await {
Ok(tool_impls) => {
for tool in tool_impls {
tools.register(tool).await;
match client.list_tools().await {
Ok(mcp_tools) => {
let tool_count = mcp_tools.len();
match client.create_tools().await {
Ok(tool_impls) => {
for tool in tool_impls {
tools.register(tool).await;
}
tracing::info!(
"Loaded {} tools from MCP server '{}'",
tool_count,
server_name
);
}
tracing::info!(
"Loaded {} tools from MCP server '{}'",
tool_count,
Err(e) => {
tracing::warn!(
"Failed to create tools from MCP server '{}': {}",
server_name,
e
);
}
}
}
Err(e) => {
let err_str = e.to_string();
if err_str.contains("401")
|| err_str.contains("authentication")
{
tracing::warn!(
"MCP server '{}' requires authentication. \
Run: ironclaw mcp auth {}",
server_name,
server_name
);
}
Err(e) => {
} else {
tracing::warn!(
"Failed to create tools from MCP server '{}': {}",
"Failed to connect to MCP server '{}': {}",
server_name,
e
);
}
}
}
Err(e) => {
let err_str = e.to_string();
if err_str.contains("401")
|| err_str.contains("authentication")
{
tracing::warn!(
"MCP server '{}' requires authentication. \
Run: ironclaw mcp auth {}",
server_name,
server_name
);
} else {
tracing::warn!(
"Failed to connect to MCP server '{}': {}",
server_name,
e
);
}
}
}
});
}
});
}
while let Some(result) = join_set.join_next().await {
if let Err(e) = result {
tracing::warn!("MCP server loading task panicked: {}", e);
while let Some(result) = join_set.join_next().await {
if let Err(e) = result {
tracing::warn!("MCP server loading task panicked: {}", e);
}
}
}
}
Err(e) => {
tracing::debug!("No MCP servers configured ({})", e);
Err(e) => {
tracing::debug!("No MCP servers configured ({})", e);
}
}
}
}
@@ -793,7 +637,6 @@ impl AppBuilder {
Ok((
mcp_session_manager,
mcp_process_manager,
wasm_tool_runtime,
extension_manager,
catalog_entries,
@@ -806,22 +649,7 @@ impl AppBuilder {
self.init_database().await?;
self.init_secrets().await?;
// Post-init validation: if a non-nearai backend was selected but
// credentials were never resolved (deferred resolution found no keys),
// fail early with a clear error instead of a confusing runtime failure.
if self.config.llm.backend != "nearai" && self.config.llm.provider.is_none() {
let backend = &self.config.llm.backend;
anyhow::bail!(
"LLM_BACKEND={backend} is configured but no credentials were found. \
Set the appropriate API key environment variable or run the setup wizard."
);
}
let (llm, cheap_llm, recording_handle) = if let Some(llm) = self.llm_override.take() {
(llm, None, None)
} else {
self.init_llm().await?
};
let (llm, cheap_llm) = self.init_llm()?;
let (safety, tools, embeddings, workspace) = self.init_tools(&llm).await?;
// Create hook registry early so runtime extension activation can register hooks.
@@ -829,7 +657,6 @@ impl AppBuilder {
let (
mcp_session_manager,
mcp_process_manager,
wasm_tool_runtime,
extension_manager,
catalog_entries,
@@ -838,31 +665,6 @@ impl AppBuilder {
// Seed workspace and backfill embeddings
if let Some(ref ws) = workspace {
// Import workspace files from disk FIRST if WORKSPACE_IMPORT_DIR is set.
// This lets Docker images / deployment scripts ship customized
// workspace templates (e.g., AGENTS.md, TOOLS.md) that override
// the generic seeds. Only imports files that don't already exist
// in the database — never overwrites user edits.
//
// Runs before seed_if_empty() so that custom templates take priority
// over generic seeds. seed_if_empty() then fills any remaining gaps.
if let Ok(import_dir) = std::env::var("WORKSPACE_IMPORT_DIR") {
let import_path = std::path::Path::new(&import_dir);
match ws.import_from_directory(import_path).await {
Ok(count) if count > 0 => {
tracing::info!("Imported {} workspace file(s) from {}", count, import_dir);
}
Ok(_) => {}
Err(e) => {
tracing::warn!(
"Failed to import workspace files from {}: {}",
import_dir,
e
);
}
}
}
match ws.seed_if_empty().await {
Ok(_) => {}
Err(e) => {
@@ -927,7 +729,6 @@ impl AppBuilder {
workspace,
extension_manager,
mcp_session_manager,
mcp_process_manager,
wasm_tool_runtime,
log_broadcaster: self.log_broadcaster,
context_manager,
@@ -935,7 +736,6 @@ impl AppBuilder {
skill_registry,
skill_catalog,
cost_guard,
recording_handle,
session: self.session,
catalog_entries,
dev_loaded_tool_names,
-251
View File
@@ -414,103 +414,10 @@ pub enum MigrationError {
Io(String),
}
// ── PID Lock ──────────────────────────────────────────────────────────────
/// Path to the PID lock file: `~/.ironclaw/ironclaw.pid`.
pub fn pid_lock_path() -> PathBuf {
ironclaw_base_dir().join("ironclaw.pid")
}
/// A PID-based lock that prevents multiple IronClaw instances from running
/// simultaneously.
///
/// Uses `fs4::try_lock_exclusive()` for atomic locking (no TOCTOU race),
/// then writes the current PID into the locked file for diagnostics.
/// The OS-level lock is held for the lifetime of this struct and
/// automatically released on drop (along with the PID file cleanup).
#[derive(Debug)]
pub struct PidLock {
path: PathBuf,
/// Held open to maintain the OS-level exclusive lock.
_file: std::fs::File,
}
/// Errors from PID lock acquisition.
#[derive(Debug, thiserror::Error)]
pub enum PidLockError {
#[error("Another IronClaw instance is already running (PID {pid})")]
AlreadyRunning { pid: u32 },
#[error("Failed to acquire PID lock: {0}")]
Io(#[from] std::io::Error),
}
impl PidLock {
/// Try to acquire the PID lock.
///
/// Uses an exclusive file lock (`flock`/`LockFileEx`) so that two
/// concurrent processes cannot both acquire the lock — no TOCTOU race.
/// If the lock file exists but the holding process is gone (stale),
/// the lock is reclaimed automatically by the OS.
pub fn acquire() -> Result<Self, PidLockError> {
Self::acquire_at(pid_lock_path())
}
/// Acquire at a specific path (for testing).
fn acquire_at(path: PathBuf) -> Result<Self, PidLockError> {
use fs4::FileExt;
use std::fs::OpenOptions;
use std::io::Write;
// Ensure parent directory exists
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
// Open (or create) the lock file
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)?;
// Try non-blocking exclusive lock — if another process holds it,
// this fails immediately instead of blocking.
if let Err(e) = file.try_lock_exclusive() {
if e.kind() == std::io::ErrorKind::WouldBlock {
// Lock held by another process — read its PID for the error message
let pid = std::fs::read_to_string(&path)
.ok()
.and_then(|s| s.trim().parse::<u32>().ok())
.unwrap_or(0);
return Err(PidLockError::AlreadyRunning { pid });
}
// Other errors (permissions, unsupported filesystem, etc.)
return Err(PidLockError::Io(e));
}
// We hold the exclusive lock — write our PID
file.set_len(0)?; // truncate
write!(file, "{}", std::process::id())?;
Ok(PidLock { path, _file: file })
}
}
impl Drop for PidLock {
fn drop(&mut self) {
// Remove the PID file; the OS-level lock is released when _file is dropped.
let _ = std::fs::remove_file(&self.path);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::process::Command;
use std::sync::Mutex;
use std::thread;
use std::time::{Duration, Instant};
use tempfile::tempdir;
static ENV_MUTEX: Mutex<()> = Mutex::new(());
@@ -1079,162 +986,4 @@ INJECTED="pwned"#;
unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") };
}
}
// ── PID Lock tests ───────────────────────────────────────────────
#[test]
fn test_pid_lock_acquire_and_drop() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
// Acquire lock
let lock = PidLock::acquire_at(pid_path.clone()).unwrap();
assert!(pid_path.exists());
// PID file should contain our PID
let contents = std::fs::read_to_string(&pid_path).unwrap();
assert_eq!(contents.trim().parse::<u32>().unwrap(), std::process::id());
// Drop should remove the file
drop(lock);
assert!(!pid_path.exists());
}
#[test]
fn test_pid_lock_rejects_second_acquire() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
// First lock succeeds
let _lock1 = PidLock::acquire_at(pid_path.clone()).unwrap();
// Second acquire on same file must fail (exclusive flock held)
let result = PidLock::acquire_at(pid_path.clone());
assert!(result.is_err());
match result.unwrap_err() {
PidLockError::AlreadyRunning { pid } => {
assert_eq!(pid, std::process::id());
}
other => panic!("expected AlreadyRunning, got: {}", other),
}
}
#[test]
fn test_pid_lock_reclaims_after_drop() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
// Acquire and release
let lock = PidLock::acquire_at(pid_path.clone()).unwrap();
drop(lock);
// Should succeed — OS lock was released on drop
let lock2 = PidLock::acquire_at(pid_path).unwrap();
drop(lock2);
}
#[test]
fn test_pid_lock_reclaims_stale_file_without_flock() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
// Write a stale PID file manually (no flock held)
std::fs::write(&pid_path, "4294967294").unwrap();
// Should succeed because no OS lock is held on the file
let lock = PidLock::acquire_at(pid_path.clone()).unwrap();
let contents = std::fs::read_to_string(&pid_path).unwrap();
assert_eq!(contents.trim().parse::<u32>().unwrap(), std::process::id());
drop(lock);
}
#[test]
fn test_pid_lock_handles_corrupt_pid_file() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
// Write garbage (no flock held)
std::fs::write(&pid_path, "not-a-number").unwrap();
// Should succeed — no OS lock held, file is reclaimed
let lock = PidLock::acquire_at(pid_path).unwrap();
drop(lock);
}
#[test]
fn test_pid_lock_creates_parent_dirs() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("nested").join("deep").join("ironclaw.pid");
let lock = PidLock::acquire_at(pid_path.clone()).unwrap();
assert!(pid_path.exists());
drop(lock);
}
#[test]
fn test_pid_lock_child_helper_holds_lock() {
if std::env::var("IRONCLAW_PID_LOCK_CHILD").ok().as_deref() != Some("1") {
return;
}
let pid_path = PathBuf::from(
std::env::var("IRONCLAW_PID_LOCK_PATH").expect("IRONCLAW_PID_LOCK_PATH missing"),
);
let hold_ms = std::env::var("IRONCLAW_PID_LOCK_HOLD_MS")
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(3000);
let _lock = PidLock::acquire_at(pid_path).expect("child failed to acquire pid lock");
thread::sleep(Duration::from_millis(hold_ms));
}
#[test]
fn test_pid_lock_rejects_lock_held_by_other_process() {
let dir = tempdir().unwrap();
let pid_path = dir.path().join("ironclaw.pid");
let current_exe = std::env::current_exe().unwrap();
let mut child = Command::new(current_exe)
.args([
"--exact",
"bootstrap::tests::test_pid_lock_child_helper_holds_lock",
"--nocapture",
"--test-threads=1",
])
.env("IRONCLAW_PID_LOCK_CHILD", "1")
.env("IRONCLAW_PID_LOCK_PATH", pid_path.display().to_string())
.env("IRONCLAW_PID_LOCK_HOLD_MS", "3000")
.spawn()
.unwrap();
let started = Instant::now();
while started.elapsed() < Duration::from_secs(2) {
if pid_path.exists() {
break;
}
if let Some(status) = child.try_wait().unwrap() {
panic!("child exited before acquiring lock: {}", status);
}
thread::sleep(Duration::from_millis(20));
}
assert!(
pid_path.exists(),
"child did not create lock file in time: {}",
pid_path.display()
);
let result = PidLock::acquire_at(pid_path.clone());
match result.unwrap_err() {
PidLockError::AlreadyRunning { .. } => {}
other => panic!("expected AlreadyRunning, got: {}", other),
}
let status = child.wait().unwrap();
assert!(status.success(), "child process failed: {}", status);
// After the child exits, lock should be released and reacquirable.
let lock = PidLock::acquire_at(pid_path).unwrap();
drop(lock);
}
}
+1 -255
View File
@@ -10,56 +10,6 @@ use uuid::Uuid;
use crate::error::ChannelError;
/// Kind of attachment carried on an incoming message.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AttachmentKind {
/// Audio content (voice notes, audio files).
Audio,
/// Image content (photos, screenshots).
Image,
/// Document content (PDFs, files).
Document,
}
impl AttachmentKind {
/// Infer attachment kind from MIME type.
pub fn from_mime_type(mime: &str) -> Self {
let base = mime.split(';').next().unwrap_or(mime).trim();
if base.starts_with("audio/") {
Self::Audio
} else if base.starts_with("image/") {
Self::Image
} else {
Self::Document
}
}
}
/// A file or media attachment on an incoming message.
#[derive(Debug, Clone)]
pub struct IncomingAttachment {
/// Unique identifier within the channel (e.g., Telegram file_id).
pub id: String,
/// What kind of content this is.
pub kind: AttachmentKind,
/// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf").
pub mime_type: String,
/// Original filename, if known.
pub filename: Option<String>,
/// File size in bytes, if known.
pub size_bytes: Option<u64>,
/// URL to download the file from the channel's API.
pub source_url: Option<String>,
/// Opaque key for host-side storage (e.g., after download/caching).
pub storage_key: Option<String>,
/// Extracted text content (e.g., OCR result, PDF text, audio transcript).
pub extracted_text: Option<String>,
/// Raw file bytes (for small files downloaded by the channel).
pub data: Vec<u8>,
/// Duration in seconds (for audio/video).
pub duration_secs: Option<u32>,
}
/// A message received from an external channel.
#[derive(Debug, Clone)]
pub struct IncomingMessage {
@@ -79,10 +29,6 @@ pub struct IncomingMessage {
pub received_at: DateTime<Utc>,
/// Channel-specific metadata.
pub metadata: serde_json::Value,
/// IANA timezone string from the client (e.g. "America/New_York").
pub timezone: Option<String>,
/// File or media attachments on this message.
pub attachments: Vec<IncomingAttachment>,
}
impl IncomingMessage {
@@ -101,8 +47,6 @@ impl IncomingMessage {
thread_id: None,
received_at: Utc::now(),
metadata: serde_json::Value::Null,
timezone: None,
attachments: Vec::new(),
}
}
@@ -123,18 +67,6 @@ impl IncomingMessage {
self.user_name = Some(name.into());
self
}
/// Set the client timezone.
pub fn with_timezone(mut self, tz: impl Into<String>) -> Self {
self.timezone = Some(tz.into());
self
}
/// Set attachments.
pub fn with_attachments(mut self, attachments: Vec<IncomingAttachment>) -> Self {
self.attachments = attachments;
self
}
}
/// Stream of incoming messages.
@@ -185,20 +117,7 @@ pub enum StatusUpdate {
/// Tool execution started.
ToolStarted { name: String },
/// Tool execution completed.
///
/// Use [`StatusUpdate::tool_completed`] to construct this variant — it
/// handles redaction of sensitive parameters and keeps the 9-line pattern
/// in one place.
ToolCompleted {
name: String,
success: bool,
/// Error message when success is false.
error: Option<String>,
/// Tool input parameters (JSON string) for display on failure.
/// Only populated when `success` is `false`. Values listed in the
/// tool's `sensitive_params()` are replaced with `"[REDACTED]"`.
parameters: Option<String>,
},
ToolCompleted { name: String, success: bool },
/// Brief preview of tool execution output.
ToolResult { name: String, preview: String },
/// Streaming text chunk.
@@ -231,45 +150,6 @@ pub enum StatusUpdate {
success: bool,
message: String,
},
/// An image was generated by a tool.
ImageGenerated {
/// Base64 data URL of the generated image.
data_url: String,
/// Optional workspace path where the image was saved.
path: Option<String>,
},
}
impl StatusUpdate {
/// Build a `ToolCompleted` status with redacted parameters.
///
/// On failure, serializes the tool's input parameters as pretty JSON after
/// replacing any keys listed in the tool's `sensitive_params()` with
/// `"[REDACTED]"`. On success, no parameters or error are included.
///
/// Pass the resolved `Tool` reference (if available) so this method can
/// query `sensitive_params()` directly — callers don't need to manage the
/// borrow lifetime of the sensitive slice.
pub fn tool_completed(
name: String,
result: &Result<String, crate::error::Error>,
params: &serde_json::Value,
tool: Option<&dyn crate::tools::Tool>,
) -> Self {
let success = result.is_ok();
let sensitive = tool.map(|t| t.sensitive_params()).unwrap_or(&[]);
Self::ToolCompleted {
name,
success,
error: result.as_ref().err().map(|e| e.to_string()),
parameters: if !success {
let safe = crate::tools::redact_params(params, sensitive);
Some(serde_json::to_string_pretty(&safe).unwrap_or_else(|_| safe.to_string()))
} else {
None
},
}
}
}
/// Trait for message channels.
@@ -343,137 +223,3 @@ pub trait Channel: Send + Sync {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Stub tool that marks `"value"` as sensitive.
struct SecretTool;
#[async_trait]
impl crate::tools::Tool for SecretTool {
fn name(&self) -> &str {
"secret_save"
}
fn description(&self) -> &str {
"stub"
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({"type": "object", "properties": {}})
}
async fn execute(
&self,
_params: serde_json::Value,
_ctx: &crate::context::JobContext,
) -> Result<crate::tools::ToolOutput, crate::tools::ToolError> {
unreachable!()
}
fn sensitive_params(&self) -> &[&str] {
&["value"]
}
}
#[test]
fn tool_completed_redacts_sensitive_params_on_failure() {
let params = serde_json::json!({"name": "api_key", "value": "sk-secret-123"});
let err: Result<String, crate::error::Error> =
Err(crate::error::ToolError::ExecutionFailed {
name: "secret_save".into(),
reason: "db error".into(),
}
.into());
let tool = SecretTool;
let status = StatusUpdate::tool_completed(
"secret_save".into(),
&err,
&params,
Some(&tool as &dyn crate::tools::Tool),
);
if let StatusUpdate::ToolCompleted {
success,
error,
parameters,
..
} = &status
{
assert!(!success);
let err_msg = error.as_deref().expect("should have error");
assert!(err_msg.contains("db error"), "error: {}", err_msg);
let param_str = parameters
.as_ref()
.expect("should have parameters on failure");
assert!(
param_str.contains("[REDACTED]"),
"sensitive value should be redacted: {}",
param_str
);
assert!(
!param_str.contains("sk-secret-123"),
"raw secret should not appear: {}",
param_str
);
assert!(
param_str.contains("api_key"),
"non-sensitive params should be preserved: {}",
param_str
);
} else {
panic!("expected ToolCompleted variant");
}
}
#[test]
fn tool_completed_no_params_on_success() {
let params = serde_json::json!({"name": "key", "value": "secret"});
let ok: Result<String, crate::error::Error> = Ok("done".into());
let status = StatusUpdate::tool_completed("secret_save".into(), &ok, &params, None);
if let StatusUpdate::ToolCompleted {
success,
error,
parameters,
..
} = &status
{
assert!(success);
assert!(error.is_none());
assert!(parameters.is_none(), "no params should be sent on success");
} else {
panic!("expected ToolCompleted variant");
}
}
#[test]
fn tool_completed_no_tool_passes_params_unredacted() {
let params = serde_json::json!({"cmd": "ls -la"});
let err: Result<String, crate::error::Error> =
Err(crate::error::ToolError::ExecutionFailed {
name: "shell".into(),
reason: "timeout".into(),
}
.into());
let status = StatusUpdate::tool_completed("shell".into(), &err, &params, None);
if let StatusUpdate::ToolCompleted { parameters, .. } = &status {
let param_str = parameters.as_ref().expect("should have parameters");
assert!(
param_str.contains("ls -la"),
"non-sensitive params should pass through: {}",
param_str
);
} else {
panic!("expected ToolCompleted variant");
}
}
#[test]
fn test_incoming_message_with_timezone() {
let msg = IncomingMessage::new("test", "user1", "hello").with_timezone("America/New_York");
assert_eq!(msg.timezone.as_deref(), Some("America/New_York"));
}
}
+6 -126
View File
@@ -17,9 +17,7 @@ use tokio::sync::{RwLock, mpsc, oneshot};
use tokio_stream::wrappers::ReceiverStream;
use uuid::Uuid;
use crate::channels::{
AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse,
};
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse};
use crate::config::HttpConfig;
use crate::error::ChannelError;
@@ -48,9 +46,8 @@ struct RateLimitState {
request_count: u32,
}
/// Maximum JSON body size for webhook requests (15 MB, to support base64 image attachments
/// with ~33% overhead from base64 encoding).
const MAX_BODY_BYTES: usize = 15 * 1024 * 1024;
/// Maximum JSON body size for webhook requests (64 KB).
const MAX_BODY_BYTES: usize = 64 * 1024;
/// Maximum number of pending wait-for-response requests.
const MAX_PENDING_RESPONSES: usize = 100;
@@ -118,34 +115,8 @@ struct WebhookRequest {
/// Whether to wait for a synchronous response.
#[serde(default)]
wait_for_response: bool,
/// Optional file attachments (base64-encoded).
#[serde(default)]
attachments: Vec<AttachmentData>,
}
/// A file attachment in a webhook request.
#[derive(Debug, Deserialize)]
struct AttachmentData {
/// MIME type (e.g. "image/png", "application/pdf").
mime_type: String,
/// Optional filename.
#[serde(default)]
filename: Option<String>,
/// Base64-encoded file data.
#[serde(default)]
data_base64: Option<String>,
/// URL to fetch the file from (not downloaded server-side for SSRF prevention).
#[serde(default)]
url: Option<String>,
}
/// Maximum size per attachment (5 MB decoded).
const MAX_ATTACHMENT_BYTES: usize = 5 * 1024 * 1024;
/// Maximum total attachment size (10 MB decoded).
const MAX_TOTAL_ATTACHMENT_BYTES: usize = 10 * 1024 * 1024;
/// Maximum number of attachments per request.
const MAX_ATTACHMENTS: usize = 5;
#[derive(Debug, Serialize)]
struct WebhookResponse {
/// Message ID assigned to this request.
@@ -240,106 +211,15 @@ async fn webhook_handler(
);
}
// Validate and decode attachments
let attachments = if !req.attachments.is_empty() {
if req.attachments.len() > MAX_ATTACHMENTS {
return (
StatusCode::BAD_REQUEST,
Json(WebhookResponse {
message_id: Uuid::nil(),
status: "error".to_string(),
response: Some(format!("Too many attachments (max {})", MAX_ATTACHMENTS)),
}),
);
}
let mut decoded_attachments = Vec::new();
let mut total_bytes: usize = 0;
for att in &req.attachments {
if let Some(ref b64) = att.data_base64 {
use base64::Engine;
let data = match base64::engine::general_purpose::STANDARD.decode(b64) {
Ok(d) => d,
Err(_) => {
return (
StatusCode::BAD_REQUEST,
Json(WebhookResponse {
message_id: Uuid::nil(),
status: "error".to_string(),
response: Some("Invalid base64 in attachment".to_string()),
}),
);
}
};
if data.len() > MAX_ATTACHMENT_BYTES {
return (
StatusCode::PAYLOAD_TOO_LARGE,
Json(WebhookResponse {
message_id: Uuid::nil(),
status: "error".to_string(),
response: Some(format!(
"Attachment too large (max {} bytes)",
MAX_ATTACHMENT_BYTES
)),
}),
);
}
total_bytes += data.len();
if total_bytes > MAX_TOTAL_ATTACHMENT_BYTES {
return (
StatusCode::PAYLOAD_TOO_LARGE,
Json(WebhookResponse {
message_id: Uuid::nil(),
status: "error".to_string(),
response: Some("Total attachment size exceeds limit".to_string()),
}),
);
}
decoded_attachments.push(IncomingAttachment {
id: Uuid::new_v4().to_string(),
kind: AttachmentKind::from_mime_type(&att.mime_type),
mime_type: att.mime_type.clone(),
filename: att.filename.clone(),
size_bytes: Some(data.len() as u64),
source_url: None,
storage_key: None,
extracted_text: None,
data,
duration_secs: None,
});
} else if let Some(ref url) = att.url {
// URL-only attachment: set source_url but don't download (SSRF prevention)
decoded_attachments.push(IncomingAttachment {
id: Uuid::new_v4().to_string(),
kind: AttachmentKind::from_mime_type(&att.mime_type),
mime_type: att.mime_type.clone(),
filename: att.filename.clone(),
size_bytes: None,
source_url: Some(url.clone()),
storage_key: None,
extracted_text: None,
data: Vec::new(),
duration_secs: None,
});
}
}
decoded_attachments
} else {
Vec::new()
};
let mut msg = IncomingMessage::new("http", &state.user_id, &req.content).with_metadata(
let msg = IncomingMessage::new("http", &state.user_id, &req.content).with_metadata(
serde_json::json!({
"wait_for_response": req.wait_for_response,
}),
);
if !attachments.is_empty() {
msg = msg.with_attachments(attachments);
}
if let Some(thread_id) = &req.thread_id {
msg = msg.with_thread(thread_id);
let msg = msg.with_thread(thread_id);
return process_message(state, msg, req.wait_for_response).await;
}
process_message(state, msg, req.wait_for_response).await
-103
View File
@@ -235,106 +235,3 @@ impl Default for ChannelManager {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::channels::IncomingMessage;
use crate::testing::StubChannel;
use futures::StreamExt;
#[tokio::test]
async fn test_add_and_start_all() {
let manager = ChannelManager::new();
let (stub, sender) = StubChannel::new("test");
manager.add(Box::new(stub)).await;
let mut stream = manager.start_all().await.expect("start_all failed");
// Inject a message through the stub
sender
.send(IncomingMessage::new("test", "user1", "hello"))
.await
.expect("send failed");
// Should appear in the merged stream
let msg = stream.next().await.expect("stream ended");
assert_eq!(msg.content, "hello");
assert_eq!(msg.channel, "test");
}
#[tokio::test]
async fn test_respond_routes_to_correct_channel() {
let manager = ChannelManager::new();
let (stub, _sender) = StubChannel::new("alpha");
// Keep a reference for response inspection
let responses = stub.captured_responses_handle();
manager.add(Box::new(stub)).await;
let msg = IncomingMessage::new("alpha", "user1", "request");
manager
.respond(&msg, OutgoingResponse::text("reply"))
.await
.expect("respond failed");
// Verify the stub captured the response
let captured = responses.lock().expect("poisoned");
assert_eq!(captured.len(), 1);
assert_eq!(captured[0].1.content, "reply");
}
#[tokio::test]
async fn test_respond_unknown_channel_errors() {
let manager = ChannelManager::new();
let msg = IncomingMessage::new("nonexistent", "user1", "test");
let result = manager.respond(&msg, OutgoingResponse::text("hi")).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_health_check_all() {
let manager = ChannelManager::new();
let (stub1, _) = StubChannel::new("healthy");
let (stub2, _) = StubChannel::new("sick");
stub2.set_healthy(false);
manager.add(Box::new(stub1)).await;
manager.add(Box::new(stub2)).await;
let results = manager.health_check_all().await;
assert!(results["healthy"].is_ok());
assert!(results["sick"].is_err());
}
#[tokio::test]
async fn test_start_all_no_channels_errors() {
let manager = ChannelManager::new();
let result = manager.start_all().await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_injection_channel_merges() {
let manager = ChannelManager::new();
let (stub, _sender) = StubChannel::new("real");
manager.add(Box::new(stub)).await;
let mut stream = manager.start_all().await.expect("start_all failed");
// Use the injection channel (simulating background task)
let inject_tx = manager.inject_sender();
inject_tx
.send(IncomingMessage::new(
"injected",
"system",
"background alert",
))
.await
.expect("inject failed");
let msg = stream.next().await.expect("stream ended");
assert_eq!(msg.content, "background alert");
}
}
+1 -4
View File
@@ -36,10 +36,7 @@ pub mod wasm;
pub mod web;
mod webhook_server;
pub use channel::{
AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse,
StatusUpdate,
};
pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
pub use http::HttpChannel;
pub use manager::ChannelManager;
pub use repl::ReplChannel;
+10 -58
View File
@@ -18,7 +18,7 @@
//! - `Esc` - Interrupt current operation
use std::borrow::Cow;
use std::io::{self, IsTerminal, Write};
use std::io::{self, Write};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -297,15 +297,10 @@ impl Channel for ReplChannel {
let esc_interrupt_triggered_for_thread = Arc::new(AtomicBool::new(false));
std::thread::spawn(move || {
let sys_tz = crate::timezone::detect_system_timezone().name().to_string();
// Single message mode: send it and return
if let Some(msg) = single_message {
let incoming = IncomingMessage::new("repl", "default", &msg).with_timezone(&sys_tz);
let incoming = IncomingMessage::new("repl", "default", &msg);
let _ = tx.blocking_send(incoming);
// Ensure the agent exits after handling exactly one turn in -m mode,
// even when other channels (gateway/http) are enabled.
let _ = tx.blocking_send(IncomingMessage::new("repl", "default", "/quit"));
return;
}
@@ -366,8 +361,7 @@ impl Channel for ReplChannel {
"/quit" | "/exit" => {
// Forward shutdown command so the agent loop exits even
// when other channels (e.g. web gateway) are still active.
let msg = IncomingMessage::new("repl", "default", "/quit")
.with_timezone(&sys_tz);
let msg = IncomingMessage::new("repl", "default", "/quit");
let _ = tx.blocking_send(msg);
break;
}
@@ -388,8 +382,7 @@ impl Channel for ReplChannel {
_ => {}
}
let msg =
IncomingMessage::new("repl", "default", line).with_timezone(&sys_tz);
let msg = IncomingMessage::new("repl", "default", line);
if tx.blocking_send(msg).is_err() {
break;
}
@@ -397,29 +390,21 @@ impl Channel for ReplChannel {
Err(ReadlineError::Interrupted) => {
if esc_interrupt_triggered_for_thread.swap(false, Ordering::Relaxed) {
// Esc: interrupt current operation and keep REPL open.
let msg = IncomingMessage::new("repl", "default", "/interrupt")
.with_timezone(&sys_tz);
let msg = IncomingMessage::new("repl", "default", "/interrupt");
if tx.blocking_send(msg).is_err() {
break;
}
} else {
// Ctrl+C (VINTR): request graceful shutdown.
let msg = IncomingMessage::new("repl", "default", "/quit")
.with_timezone(&sys_tz);
let msg = IncomingMessage::new("repl", "default", "/quit");
let _ = tx.blocking_send(msg);
break;
}
}
Err(ReadlineError::Eof) => {
// Ctrl+D in interactive mode: graceful shutdown.
// In daemon mode (stdin = /dev/null, no TTY), EOF arrives
// immediately — just drop the REPL thread silently so other
// channels (gateway, telegram, …) keep running.
if std::io::stdin().is_terminal() {
let msg = IncomingMessage::new("repl", "default", "/quit")
.with_timezone(&sys_tz);
let _ = tx.blocking_send(msg);
}
// Ctrl+D: send /quit so the agent loop runs graceful shutdown
let msg = IncomingMessage::new("repl", "default", "/quit");
let _ = tx.blocking_send(msg);
break;
}
Err(e) => {
@@ -481,7 +466,7 @@ impl Channel for ReplChannel {
StatusUpdate::ToolStarted { name } => {
eprintln!(" \x1b[33m\u{25CB} {name}\x1b[0m");
}
StatusUpdate::ToolCompleted { name, success, .. } => {
StatusUpdate::ToolCompleted { name, success } => {
if success {
eprintln!(" \x1b[32m\u{25CF} {name}\x1b[0m");
} else {
@@ -600,13 +585,6 @@ impl Channel for ReplChannel {
eprintln!("\x1b[31m {extension_name}: {message}\x1b[0m");
}
}
StatusUpdate::ImageGenerated { path, .. } => {
if let Some(ref p) = path {
eprintln!("\x1b[36m [image] {p}\x1b[0m");
} else {
eprintln!("\x1b[36m [image generated]\x1b[0m");
}
}
}
Ok(())
}
@@ -636,29 +614,3 @@ impl Channel for ReplChannel {
Ok(())
}
}
#[cfg(test)]
mod tests {
use futures::StreamExt;
use super::*;
#[tokio::test]
async fn single_message_mode_sends_message_then_quit() {
let repl = ReplChannel::with_message("hi".to_string());
let mut stream = repl.start().await.expect("repl start should succeed");
let first = stream.next().await.expect("first message missing");
assert_eq!(first.channel, "repl");
assert_eq!(first.content, "hi");
let second = stream.next().await.expect("quit message missing");
assert_eq!(second.channel, "repl");
assert_eq!(second.content, "/quit");
assert!(
stream.next().await.is_none(),
"stream should end after /quit"
);
}
}
+1 -1
View File
@@ -974,7 +974,7 @@ impl Channel for SignalChannel {
// Send tool completed notification (debug mode only)
if self.is_debug()
&& let StatusUpdate::ToolCompleted { name, success, .. } = &status
&& let StatusUpdate::ToolCompleted { name, success } = &status
&& let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str())
{
let (icon, color) = if *success {
-3
View File
@@ -80,9 +80,6 @@ pub enum WasmChannelError {
#[error("HTTP request error: {0}")]
HttpRequest(String),
#[error("WIT version mismatch: {0}")]
IncompatibleWitVersion(String),
}
impl From<crate::tools::wasm::WasmError> for WasmChannelError {
+1 -329
View File
@@ -5,7 +5,6 @@
//! - Workspace write access (scoped to channel namespace)
//! - Rate limiting for message emission
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig};
@@ -18,52 +17,6 @@ const MAX_EMITS_PER_EXECUTION: usize = 100;
/// Maximum message content size (64 KB).
const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024;
/// A file or media attachment on an incoming message.
#[derive(Debug, Clone)]
pub struct Attachment {
/// Unique identifier within the channel (e.g., Telegram file_id).
pub id: String,
/// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf").
pub mime_type: String,
/// Original filename, if known.
pub filename: Option<String>,
/// File size in bytes, if known.
pub size_bytes: Option<u64>,
/// URL to download the file from the channel's API.
pub source_url: Option<String>,
/// Opaque key for host-side storage (e.g., after download/caching).
pub storage_key: Option<String>,
/// Extracted text content (e.g., OCR result, PDF text, audio transcript).
pub extracted_text: Option<String>,
/// Raw file bytes (for small files downloaded by the channel).
pub data: Vec<u8>,
/// Duration in seconds (for audio/video).
pub duration_secs: Option<u32>,
}
/// Maximum total attachment size per message (20 MB).
const MAX_ATTACHMENT_TOTAL_SIZE: u64 = 20 * 1024 * 1024;
/// Maximum number of attachments per message.
const MAX_ATTACHMENTS_PER_MESSAGE: usize = 10;
/// Allowed MIME type prefixes for attachments.
const ALLOWED_MIME_PREFIXES: &[&str] = &[
"image/",
"audio/",
"video/",
"application/pdf",
"application/vnd.",
"application/msword",
"application/rtf",
"text/",
"application/json",
"application/zip",
"application/gzip",
"application/x-tar",
"application/octet-stream",
];
/// A message emitted by a WASM channel to be sent to the agent.
#[derive(Debug, Clone)]
pub struct EmittedMessage {
@@ -82,9 +35,6 @@ pub struct EmittedMessage {
/// Channel-specific metadata as JSON string.
pub metadata_json: String,
/// File or media attachments on this message.
pub attachments: Vec<Attachment>,
/// Timestamp when the message was emitted.
pub emitted_at_millis: u64,
}
@@ -98,7 +48,6 @@ impl EmittedMessage {
content: content.into(),
thread_id: None,
metadata_json: "{}".to_string(),
attachments: Vec::new(),
emitted_at_millis: SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
@@ -123,12 +72,6 @@ impl EmittedMessage {
self.metadata_json = metadata_json.into();
self
}
/// Set attachments.
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
self.attachments = attachments;
self
}
}
/// A pending workspace write operation.
@@ -169,13 +112,6 @@ pub struct ChannelHostState {
/// Count of emits dropped due to rate limiting.
emits_dropped: usize,
/// Binary data stored for attachments via `store-attachment-data`.
/// Keyed by attachment ID, cleared after callback completes.
attachment_data: HashMap<String, Vec<u8>>,
/// Total bytes stored in attachment_data (for enforcing limits).
attachment_data_total: u64,
}
impl std::fmt::Debug for ChannelHostState {
@@ -205,8 +141,6 @@ impl ChannelHostState {
emit_count: 0,
emit_enabled: true,
emits_dropped: 0,
attachment_data: HashMap::new(),
attachment_data_total: 0,
}
}
@@ -234,7 +168,6 @@ impl ChannelHostState {
///
/// Messages are queued and delivered after callback execution completes.
/// Rate limiting is enforced per-execution and globally.
/// Attachments are validated for count, total size, and MIME type.
pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> {
// Check per-execution limit
if !self.emit_enabled {
@@ -253,9 +186,6 @@ impl ChannelHostState {
return Ok(());
}
// Validate attachments
let msg = self.validate_attachments(msg);
// Validate message content size
if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE {
tracing::warn!(
@@ -279,71 +209,6 @@ impl ChannelHostState {
Ok(())
}
/// Validate and sanitize attachments on an emitted message.
///
/// Enforces count limits, total size limits, and MIME type allowlist.
/// Invalid attachments are dropped with a warning.
fn validate_attachments(&self, mut msg: EmittedMessage) -> EmittedMessage {
if msg.attachments.is_empty() {
return msg;
}
// Enforce attachment count limit
if msg.attachments.len() > MAX_ATTACHMENTS_PER_MESSAGE {
tracing::warn!(
channel = %self.channel_name,
count = msg.attachments.len(),
max = MAX_ATTACHMENTS_PER_MESSAGE,
"Too many attachments, truncating"
);
msg.attachments.truncate(MAX_ATTACHMENTS_PER_MESSAGE);
}
// Filter by MIME type and enforce total size limit
let mut total_size: u64 = 0;
msg.attachments.retain(|att| {
let mime_ok = ALLOWED_MIME_PREFIXES
.iter()
.any(|prefix| att.mime_type.starts_with(prefix));
if !mime_ok {
tracing::warn!(
channel = %self.channel_name,
mime_type = %att.mime_type,
"Attachment MIME type not allowed, dropping"
);
return false;
}
// Use the larger of reported size_bytes and actual stored data size
// to prevent WASM channels from under-reporting to bypass limits.
let stored_size = self
.attachment_data
.get(&att.id)
.map(|d| d.len() as u64)
.unwrap_or(att.data.len() as u64);
let size = att
.size_bytes
.map(|reported| reported.max(stored_size))
.unwrap_or(stored_size);
if size > 0 {
total_size = total_size.saturating_add(size);
if total_size > MAX_ATTACHMENT_TOTAL_SIZE {
tracing::warn!(
channel = %self.channel_name,
total_size,
max = MAX_ATTACHMENT_TOTAL_SIZE,
"Attachment total size exceeded, dropping"
);
return false;
}
}
true
});
msg
}
/// Take all emitted messages (clears the queue).
pub fn take_emitted_messages(&mut self) -> Vec<EmittedMessage> {
std::mem::take(&mut self.emitted_messages)
@@ -359,69 +224,6 @@ impl ChannelHostState {
self.emits_dropped
}
/// Store binary data for an attachment.
///
/// Called by WASM channels to associate downloaded bytes with an attachment ID.
/// The data is retrieved after callback completion and merged into `Attachment::data`.
pub fn store_attachment_data(
&mut self,
attachment_id: &str,
data: Vec<u8>,
) -> Result<(), WasmChannelError> {
const MAX_PER_ATTACHMENT: u64 = 20 * 1024 * 1024; // 20 MB
const MAX_TOTAL: u64 = 50 * 1024 * 1024; // 50 MB
let size = data.len() as u64;
if size > MAX_PER_ATTACHMENT {
return Err(WasmChannelError::CallbackFailed {
name: self.channel_name.clone(),
reason: format!(
"Attachment data too large: {} bytes (max {})",
size, MAX_PER_ATTACHMENT
),
});
}
// Subtract the old entry size (if overwriting) before adding new size
let old_size = self
.attachment_data
.get(attachment_id)
.map(|d| d.len() as u64)
.unwrap_or(0);
let adjusted_total = self.attachment_data_total.saturating_sub(old_size);
let new_total = adjusted_total.saturating_add(size);
if new_total > MAX_TOTAL {
return Err(WasmChannelError::CallbackFailed {
name: self.channel_name.clone(),
reason: format!(
"Total attachment data too large: {} bytes (max {})",
new_total, MAX_TOTAL
),
});
}
self.attachment_data_total = new_total;
self.attachment_data.insert(attachment_id.to_string(), data);
Ok(())
}
/// Remove stored binary data for a specific attachment ID.
pub fn remove_attachment_data(&mut self, id: &str) -> Option<Vec<u8>> {
if let Some(data) = self.attachment_data.remove(id) {
self.attachment_data_total =
self.attachment_data_total.saturating_sub(data.len() as u64);
Some(data)
} else {
None
}
}
/// Take all stored attachment data (clears the store).
pub fn take_attachment_data(&mut self) -> HashMap<String, Vec<u8>> {
self.attachment_data_total = 0;
std::mem::take(&mut self.attachment_data)
}
/// Write to workspace (scoped to channel namespace).
///
/// Writes are queued and committed after callback execution completes.
@@ -629,8 +431,7 @@ impl ChannelEmitRateLimiter {
mod tests {
use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig};
use crate::channels::wasm::host::{
Attachment, ChannelEmitRateLimiter, ChannelHostState, EmittedMessage,
MAX_ATTACHMENT_TOTAL_SIZE, MAX_ATTACHMENTS_PER_MESSAGE, MAX_EMITS_PER_EXECUTION,
ChannelEmitRateLimiter, ChannelHostState, EmittedMessage, MAX_EMITS_PER_EXECUTION,
};
#[test]
@@ -959,133 +760,4 @@ mod tests {
Some("200".to_string())
);
}
// === Attachment validation tests ===
fn make_attachment(id: &str, mime: &str, size: Option<u64>) -> Attachment {
Attachment {
id: id.to_string(),
mime_type: mime.to_string(),
filename: None,
size_bytes: size,
source_url: None,
storage_key: None,
extracted_text: None,
data: Vec::new(),
duration_secs: None,
}
}
#[test]
fn test_emit_message_with_attachments() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
let msg = EmittedMessage::new("user1", "Check this image")
.with_attachments(vec![make_attachment("file1", "image/jpeg", Some(1024))]);
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
assert_eq!(messages.len(), 1);
assert_eq!(messages[0].attachments.len(), 1);
assert_eq!(messages[0].attachments[0].id, "file1");
assert_eq!(messages[0].attachments[0].mime_type, "image/jpeg");
assert_eq!(messages[0].attachments[0].size_bytes, Some(1024));
}
#[test]
fn test_emit_message_no_attachments_backward_compat() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
let msg = EmittedMessage::new("user1", "Just text");
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
assert_eq!(messages.len(), 1);
assert!(messages[0].attachments.is_empty());
}
#[test]
fn test_attachment_count_limit() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
let attachments: Vec<Attachment> = (0..MAX_ATTACHMENTS_PER_MESSAGE + 5)
.map(|i| make_attachment(&format!("file{}", i), "image/png", Some(100)))
.collect();
let msg = EmittedMessage::new("user1", "Many files").with_attachments(attachments);
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
assert_eq!(messages[0].attachments.len(), MAX_ATTACHMENTS_PER_MESSAGE);
}
#[test]
fn test_attachment_total_size_limit() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
// Each file is 1/3 of the limit, so 3 fit but 4th does not
let chunk_size = MAX_ATTACHMENT_TOTAL_SIZE / 3;
let attachments = vec![
make_attachment("file1", "image/png", Some(chunk_size)),
make_attachment("file2", "image/png", Some(chunk_size)),
make_attachment("file3", "image/png", Some(chunk_size)),
make_attachment("file4", "image/png", Some(chunk_size)),
];
let msg = EmittedMessage::new("user1", "Big files").with_attachments(attachments);
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
// Only first 3 fit within the total size limit
assert_eq!(messages[0].attachments.len(), 3);
}
#[test]
fn test_attachment_mime_type_filtering() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
let attachments = vec![
make_attachment("ok1", "image/jpeg", Some(100)),
make_attachment("bad1", "application/x-executable", Some(100)),
make_attachment("ok2", "application/pdf", Some(100)),
make_attachment("bad2", "application/x-msdos-program", Some(100)),
make_attachment("ok3", "text/plain", Some(100)),
make_attachment("ok4", "audio/mpeg", Some(100)),
make_attachment("ok5", "video/mp4", Some(100)),
];
let msg = EmittedMessage::new("user1", "Mixed files").with_attachments(attachments);
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
let ids: Vec<&str> = messages[0]
.attachments
.iter()
.map(|a| a.id.as_str())
.collect();
assert_eq!(ids, vec!["ok1", "ok2", "ok3", "ok4", "ok5"]);
}
#[test]
fn test_attachment_unknown_size_allowed() {
let caps = ChannelCapabilities::for_channel("test");
let mut state = ChannelHostState::new("test", caps);
let attachments = vec![
make_attachment("file1", "image/jpeg", None),
make_attachment("file2", "image/png", None),
];
let msg = EmittedMessage::new("user1", "No sizes").with_attachments(attachments);
state.emit_message(msg).unwrap();
let messages = state.take_emitted_messages();
assert_eq!(messages[0].attachments.len(), 2);
}
}
+1 -29
View File
@@ -19,14 +19,12 @@ use crate::channels::wasm::schema::ChannelCapabilitiesFile;
use crate::channels::wasm::wrapper::WasmChannel;
use crate::db::SettingsStore;
use crate::pairing::PairingStore;
use crate::secrets::SecretsStore;
/// Loads WASM channels from the filesystem.
pub struct WasmChannelLoader {
runtime: Arc<WasmChannelRuntime>,
pairing_store: Arc<PairingStore>,
settings_store: Option<Arc<dyn SettingsStore>>,
secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>,
}
impl WasmChannelLoader {
@@ -40,16 +38,9 @@ impl WasmChannelLoader {
runtime,
pairing_store,
settings_store,
secrets_store: None,
}
}
/// Set the secrets store for host-based credential injection in WASM channels.
pub fn with_secrets_store(mut self, store: Arc<dyn SecretsStore + Send + Sync>) -> Self {
self.secrets_store = Some(store);
self
}
/// Load a single WASM channel from a file pair.
///
/// Expects:
@@ -81,7 +72,6 @@ impl WasmChannelLoader {
let cap_bytes = fs::read(cap_path).await?;
let cap_file = ChannelCapabilitiesFile::from_bytes(&cap_bytes)
.map_err(|e| WasmChannelError::InvalidCapabilities(e.to_string()))?;
cap_file.validate();
// Debug: log raw capabilities
tracing::debug!(
@@ -90,14 +80,6 @@ impl WasmChannelLoader {
"Parsed capabilities file"
);
// Check WIT version compatibility
crate::tools::wasm::loader::check_wit_version_compat(
name,
cap_file.wit_version.as_deref(),
crate::tools::wasm::WIT_CHANNEL_VERSION,
)
.map_err(|e| WasmChannelError::IncompatibleWitVersion(e.to_string()))?;
let caps = cap_file.to_capabilities();
// Debug: log resulting capabilities
@@ -145,7 +127,7 @@ impl WasmChannelLoader {
.await?;
// Create the channel
let mut channel = WasmChannel::new(
let channel = WasmChannel::new(
self.runtime.clone(),
prepared,
capabilities,
@@ -153,9 +135,6 @@ impl WasmChannelLoader {
self.pairing_store.clone(),
self.settings_store.clone(),
);
if let Some(ref secrets) = self.secrets_store {
channel = channel.with_secrets_store(Arc::clone(secrets));
}
tracing::info!(
name = name,
@@ -285,13 +264,6 @@ impl LoadedChannel {
.and_then(|f| f.signature_key_secret_name().map(|s| s.to_string()))
}
/// Get the HMAC-SHA256 signing secret name from capabilities.
pub fn hmac_secret_name(&self) -> Option<String> {
self.capabilities_file
.as_ref()
.and_then(|f| f.hmac_secret_name().map(|s| s.to_string()))
}
/// Get the webhook secret name from capabilities.
pub fn webhook_secret_name(&self) -> String {
self.capabilities_file
-2
View File
@@ -87,8 +87,6 @@ mod router;
mod runtime;
mod schema;
pub(crate) mod signature;
#[allow(dead_code)]
pub(crate) mod storage;
mod wrapper;
// Core types
+1 -337
View File
@@ -44,8 +44,6 @@ pub struct WasmChannelRouter {
secret_headers: RwLock<HashMap<String, String>>,
/// Ed25519 public keys for signature verification by channel name (hex-encoded).
signature_keys: RwLock<HashMap<String, String>>,
/// HMAC-SHA256 signing secrets for signature verification by channel name (Slack-style).
hmac_secrets: RwLock<HashMap<String, String>>,
}
impl WasmChannelRouter {
@@ -57,7 +55,6 @@ impl WasmChannelRouter {
secrets: RwLock::new(HashMap::new()),
secret_headers: RwLock::new(HashMap::new()),
signature_keys: RwLock::new(HashMap::new()),
hmac_secrets: RwLock::new(HashMap::new()),
}
}
@@ -137,7 +134,6 @@ impl WasmChannelRouter {
self.secrets.write().await.remove(channel_name);
self.secret_headers.write().await.remove(channel_name);
self.signature_keys.write().await.remove(channel_name);
self.hmac_secrets.write().await.remove(channel_name);
// Remove all paths for this channel
self.path_to_channel
@@ -212,24 +208,6 @@ impl WasmChannelRouter {
pub async fn get_signature_key(&self, channel_name: &str) -> Option<String> {
self.signature_keys.read().await.get(channel_name).cloned()
}
/// Register an HMAC-SHA256 signing secret for signature verification.
///
/// Channels with a registered secret will have Slack-style HMAC-SHA256
/// signature validation performed before forwarding to WASM.
pub async fn register_hmac_secret(&self, channel_name: &str, secret: &str) {
self.hmac_secrets
.write()
.await
.insert(channel_name.to_string(), secret.to_string());
}
/// Get the HMAC signing secret for a channel.
///
/// Returns `None` if no secret is registered (no HMAC check needed).
pub async fn get_hmac_secret(&self, channel_name: &str) -> Option<String> {
self.hmac_secrets.read().await.get(channel_name).cloned()
}
}
impl Default for WasmChannelRouter {
@@ -449,57 +427,6 @@ async fn webhook_handler(
}
}
// HMAC-SHA256 signature verification (Slack-style)
if let Some(hmac_secret) = state.router.get_hmac_secret(channel_name).await {
let timestamp = headers
.get("x-slack-request-timestamp")
.and_then(|v| v.to_str().ok());
let sig_header = headers
.get("x-slack-signature")
.and_then(|v| v.to_str().ok());
match (timestamp, sig_header) {
(Some(ts), Some(sig)) => {
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs() as i64;
if !crate::channels::wasm::signature::verify_slack_signature(
&hmac_secret,
ts,
&body,
sig,
now_secs,
) {
tracing::warn!(
channel = %channel_name,
"HMAC-SHA256 signature verification failed"
);
return (
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Invalid Slack signature"
})),
);
}
tracing::debug!(channel = %channel_name, "HMAC-SHA256 signature verified");
}
_ => {
tracing::warn!(
channel = %channel_name,
"Slack signature headers missing but secret is registered"
);
return (
StatusCode::UNAUTHORIZED,
Json(serde_json::json!({
"error": "Missing Slack signature headers"
})),
);
}
}
}
// Convert headers to HashMap
let headers_map: HashMap<String, String> = headers
.iter()
@@ -804,59 +731,7 @@ mod tests {
assert_eq!(router.get_secret_header("slack").await, "X-Webhook-Secret");
}
// ── Category 3: Router HMAC Secret Management ───────────────────────
#[tokio::test]
async fn test_register_and_get_hmac_secret() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("slack");
router.register(channel, vec![], None, None).await;
let hmac_secret = "my-slack-signing-secret";
router.register_hmac_secret("slack", hmac_secret).await;
let retrieved = router.get_hmac_secret("slack").await;
assert_eq!(retrieved, Some(hmac_secret.to_string()));
}
#[tokio::test]
async fn test_no_hmac_secret_returns_none() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("slack");
router.register(channel, vec![], None, None).await;
// Slack has no HMAC secret registered
let secret = router.get_hmac_secret("slack").await;
assert!(secret.is_none());
}
#[tokio::test]
async fn test_unregister_removes_hmac_secret() {
let router = WasmChannelRouter::new();
let channel = create_test_channel("slack");
let endpoints = vec![RegisteredEndpoint {
channel_name: "slack".to_string(),
path: "/webhook/slack".to_string(),
methods: vec!["POST".to_string()],
require_secret: false,
}];
router.register(channel, endpoints, None, None).await;
router.register_hmac_secret("slack", "signing-secret").await;
// Secret should exist
assert!(router.get_hmac_secret("slack").await.is_some());
// Unregister
router.unregister("slack").await;
// Secret should be gone
assert!(router.get_hmac_secret("slack").await.is_none());
}
// ── Category 4: Router Signature Key Management ─────────────────────
// ── Category 3: Router Signature Key Management ─────────────────────
#[tokio::test]
async fn test_register_and_get_signature_key() {
@@ -1288,215 +1163,4 @@ mod tests {
"Valid secret + valid signature should not return 401"
);
}
// ── HMAC-SHA256 Webhook Signature Tests ────────────────────────────
/// Helper to create a router with a registered channel at /webhook/slack.
async fn setup_slack_router() -> (Arc<WasmChannelRouter>, AxumRouter) {
let wasm_router = Arc::new(WasmChannelRouter::new());
let channel = create_test_channel("slack");
let endpoints = vec![RegisteredEndpoint {
channel_name: "slack".to_string(),
path: "/webhook/slack".to_string(),
methods: vec!["POST".to_string()],
require_secret: false,
}];
wasm_router.register(channel, endpoints, None, None).await;
let app = create_wasm_channel_router(wasm_router.clone(), None);
(wasm_router, app)
}
/// Helper: compute expected Slack signature for testing.
fn slack_signature(signing_secret: &str, timestamp: &str, body: &[u8]) -> String {
use hmac::{Hmac, Mac};
use sha2::Sha256;
let mut basestring = Vec::new();
basestring.extend_from_slice(b"v0:");
basestring.extend_from_slice(timestamp.as_bytes());
basestring.push(b':');
basestring.extend_from_slice(body);
let mut mac = Hmac::<Sha256>::new_from_slice(signing_secret.as_bytes()).unwrap();
mac.update(&basestring);
let computed = mac.finalize().into_bytes();
format!("v0={}", hex::encode(computed))
}
#[tokio::test]
async fn test_webhook_hmac_rejects_missing_sig_headers() {
let (wasm_router, app) = setup_slack_router().await;
wasm_router
.register_hmac_secret("slack", "my-signing-secret")
.await;
// Send request without HMAC signature headers
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G"))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Missing HMAC signature headers should return 401"
);
}
#[tokio::test]
async fn test_webhook_hmac_rejects_invalid_signature() {
let (wasm_router, app) = setup_slack_router().await;
wasm_router
.register_hmac_secret("slack", "my-signing-secret")
.await;
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.header("x-slack-request-timestamp", "1234567890")
.header("x-slack-signature", "v0=deadbeefdeadbeef")
.body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G"))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Invalid HMAC signature should return 401"
);
}
#[tokio::test]
async fn test_webhook_hmac_accepts_valid_signature() {
let (wasm_router, app) = setup_slack_router().await;
let signing_secret = "my-signing-secret";
wasm_router
.register_hmac_secret("slack", signing_secret)
.await;
let now_secs = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let timestamp = now_secs.to_string();
let body = b"token=xyzz0WbapA4vBCDEFasx0q6G";
let signature = slack_signature(signing_secret, &timestamp, body);
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.header("x-slack-request-timestamp", &timestamp)
.header("x-slack-signature", &signature)
.body(Body::from(&body[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should NOT be 401 — signature is valid (may be 500 since no WASM module)
assert_ne!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Valid HMAC signature should not return 401"
);
}
#[tokio::test]
async fn test_webhook_hmac_skips_check_for_no_secret() {
let (_wasm_router, app) = setup_slack_router().await;
// No HMAC secret registered — should not require signature
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G"))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should NOT be 401 (may be 500 since no WASM module, but not auth failure)
assert_ne!(
resp.status(),
StatusCode::UNAUTHORIZED,
"No HMAC secret registered — should skip check"
);
}
#[tokio::test]
async fn test_webhook_hmac_uses_correct_body() {
let (wasm_router, app) = setup_slack_router().await;
let signing_secret = "my-signing-secret";
wasm_router
.register_hmac_secret("slack", signing_secret)
.await;
let timestamp = "1234567890";
let body_a = b"token=xyzz0WbapA4vBCDEFasx0q6G";
let body_b = b"token=MODIFIED";
// Sign body A
let signature = slack_signature(signing_secret, timestamp, body_a);
// But send body B
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.header("x-slack-request-timestamp", timestamp)
.header("x-slack-signature", &signature)
.body(Body::from(&body_b[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Signature for different body should return 401"
);
}
#[tokio::test]
async fn test_webhook_hmac_uses_correct_timestamp() {
let (wasm_router, app) = setup_slack_router().await;
let signing_secret = "my-signing-secret";
wasm_router
.register_hmac_secret("slack", signing_secret)
.await;
let timestamp_a = "1234567890";
let timestamp_b = "9999999999";
let body = b"token=xyzz0WbapA4vBCDEFasx0q6G";
// Sign with timestamp A
let signature = slack_signature(signing_secret, timestamp_a, body);
// But send timestamp B in the header
let req = Request::builder()
.method("POST")
.uri("/webhook/slack")
.header("content-type", "application/json")
.header("x-slack-request-timestamp", timestamp_b)
.header("x-slack-signature", &signature)
.body(Body::from(&body[..]))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(
resp.status(),
StatusCode::UNAUTHORIZED,
"Signature with mismatched timestamp should return 401"
);
}
}
+1 -10
View File
@@ -153,16 +153,7 @@ impl WasmChannelRuntime {
// Enable persistent compilation cache. Wasmtime serializes compiled native
// code to disk (~/.cache/wasmtime by default), so subsequent startups
// deserialize instead of recompiling — typically 10-50x faster.
//
// On Windows, each Engine gets its own cache subdirectory to avoid
// OS error 33 (ERROR_LOCK_VIOLATION) when multiple engines share the
// default cache and Windows holds exclusive locks on memory-mapped
// files. See #448.
if let Err(e) = crate::tools::wasm::enable_compilation_cache(
&mut wasmtime_config,
"channels",
config.cache_dir.as_deref(),
) {
if let Err(e) = wasmtime_config.cache_config_load_default() {
tracing::warn!("Failed to enable wasmtime compilation cache: {}", e);
}
-118
View File
@@ -51,14 +51,6 @@ use crate::tools::wasm::{CapabilitiesFile as ToolCapabilitiesFile, RateLimitSche
/// Root schema for a channel capabilities JSON file.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ChannelCapabilitiesFile {
/// Extension version (semver).
#[serde(default)]
pub version: Option<String>,
/// WIT interface version this channel was compiled against (semver).
#[serde(default)]
pub wit_version: Option<String>,
/// File type, must be "channel".
#[serde(default = "default_type")]
pub r#type: String,
@@ -98,37 +90,6 @@ impl ChannelCapabilitiesFile {
serde_json::from_slice(bytes)
}
/// Validate the capabilities file and emit warnings for common misconfigurations.
///
/// Called once at load time to catch issues early. Warnings are emitted via
/// `tracing::warn` so they show up in startup logs without blocking loading.
pub fn validate(&self) {
const MIN_PROMPT_LENGTH: usize = 30;
// Check for short prompts in required_secrets
for secret in &self.setup.required_secrets {
if secret.prompt.len() < MIN_PROMPT_LENGTH {
tracing::warn!(
channel = self.name,
secret = secret.name,
prompt = secret.prompt,
"setup.required_secrets prompt is shorter than {} chars — \
consider a more descriptive prompt that tells the user where to find this value",
MIN_PROMPT_LENGTH
);
}
}
// Has required_secrets but no setup_url
if !self.setup.required_secrets.is_empty() && self.setup.setup_url.is_none() {
tracing::warn!(
channel = self.name,
"setup.required_secrets defined but no setup.setup_url — \
user has no link to obtain credentials"
);
}
}
/// Convert to runtime ChannelCapabilities.
pub fn to_capabilities(&self) -> ChannelCapabilities {
self.capabilities.to_channel_capabilities(&self.name)
@@ -162,18 +123,6 @@ impl ChannelCapabilitiesFile {
.and_then(|w| w.signature_key_secret_name.as_deref())
}
/// Get the HMAC-SHA256 signing secret name for this channel.
///
/// Returns the secret name declared in `webhook.hmac_secret_name`,
/// used to look up the HMAC signing secret in the secrets store (Slack-style).
pub fn hmac_secret_name(&self) -> Option<&str> {
self.capabilities
.channel
.as_ref()
.and_then(|c| c.webhook.as_ref())
.and_then(|w| w.hmac_secret_name.as_deref())
}
/// Get the webhook secret name for this channel.
///
/// Returns the configured secret name or defaults to "{channel_name}_webhook_secret".
@@ -298,10 +247,6 @@ pub struct WebhookSchema {
/// for signature verification (e.g., Discord interaction verification).
#[serde(default)]
pub signature_key_secret_name: Option<String>,
/// Secret name in secrets store for HMAC-SHA256 signing (Slack-style).
#[serde(default)]
pub hmac_secret_name: Option<String>,
}
/// Setup configuration schema.
@@ -317,10 +262,6 @@ pub struct SetupSchema {
/// Placeholders like {secret_name} are replaced with actual values.
#[serde(default)]
pub validation_endpoint: Option<String>,
/// User-facing URL where they can create/manage credentials.
#[serde(default)]
pub setup_url: Option<String>,
}
/// Configuration for a secret required during setup.
@@ -664,65 +605,6 @@ mod tests {
// ── Category 5: Discord Capabilities Setup & Configuration ──────────
#[test]
fn test_validate_channel_short_prompt() {
// prompt < 30 chars — should not panic
let json = r#"{
"name": "test-channel",
"setup": {
"required_secrets": [
{ "name": "bot_token", "prompt": "Bot token" }
],
"setup_url": "https://example.com"
}
}"#;
let file = ChannelCapabilitiesFile::from_json(json).unwrap();
// Should not panic; warning emitted for short prompt
file.validate();
}
#[test]
fn test_validate_channel_missing_setup_url() {
// required_secrets without setup_url — should not panic
let json = r#"{
"name": "test-channel",
"setup": {
"required_secrets": [
{
"name": "bot_token",
"prompt": "Enter your bot token from the developer portal settings"
}
]
}
}"#;
let file = ChannelCapabilitiesFile::from_json(json).unwrap();
// Should not panic; warning emitted for missing setup_url
file.validate();
}
#[test]
fn test_validate_clean_channel() {
// Well-configured channel — should not panic or warn
let json = r#"{
"name": "good-channel",
"setup": {
"required_secrets": [
{
"name": "bot_token",
"prompt": "Enter your bot token from https://example.com/bot-settings"
}
],
"setup_url": "https://example.com/bot-settings"
}
}"#;
let file = ChannelCapabilitiesFile::from_json(json).unwrap();
// Should not panic and emits no warnings
file.validate();
}
#[test]
fn test_discord_capabilities_has_public_key_secret() {
let json = include_str!("../../../channels-src/discord/discord.capabilities.json");

Some files were not shown because too many files have changed in this diff Show More