diff --git a/.claude/commands/add-sse-event.md b/.claude/commands/add-sse-event.md index 7215a48e..23f47a08 100644 --- a/.claude/commands/add-sse-event.md +++ b/.claude/commands/add-sse-event.md @@ -64,7 +64,7 @@ If the event needs custom UI (cards, badges, etc.), add styles. Follow the exist Identify where in the backend this event should be triggered. Common locations: - `src/agent/agent_loop.rs` - During message processing or tool execution -- `src/agent/worker.rs` - During job execution +- `src/worker/job.rs` - During job execution - `src/agent/heartbeat.rs` - During periodic execution Use the existing pattern: diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..4fc7cbf2 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,50 @@ +## Summary + + + +- + +## Change Type + + + +- [ ] Bug fix +- [ ] New feature +- [ ] Refactor +- [ ] Documentation +- [ ] CI/Infrastructure +- [ ] Security +- [ ] Dependencies + +## Linked Issue + + + +## Validation + + + +- [ ] `cargo fmt` +- [ ] `cargo clippy --all --benches --tests --examples --all-features` +- [ ] Relevant tests pass: +- [ ] Manual testing: + +## Security Impact + + + +## Database Impact + + + +## Blast Radius + + + +## Rollback Plan + + + +--- + +**Review track**: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 34eb554d..62e5eae6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -144,6 +144,8 @@ jobs: - name: Patch manifests with WASM checksums if: ${{ needs.plan.outputs.publishing == 'true' }} shell: bash + env: + RELEASE_TAG: ${{ github.ref_name }} run: | CHECKSUMS="target/distrib/checksums.txt" if [ ! -f "$CHECKSUMS" ]; then @@ -154,12 +156,17 @@ jobs: while IFS= read -r line; do sha256=$(echo "$line" | awk '{print $1}') filename=$(echo "$line" | awk '{print $2}') - name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//') + # Strip -{version}-wasm32-wasip2.tar.gz to get the extension name. + # Use '.*' (greedy) so pre-release suffixes like -alpha.1 are consumed too. + name=$(echo "$filename" | sed 's/-[0-9].*-wasm32-wasip2\.tar\.gz$//') + url="https://github.com/nearai/ironclaw/releases/download/${RELEASE_TAG}/${filename}" for manifest in registry/tools/${name}.json registry/channels/${name}.json; do if [ -f "$manifest" ]; then - jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" - echo "Patched $manifest with sha256=$sha256" + jq --arg sha "$sha256" --arg url "$url" \ + '.artifacts["wasm32-wasip2"].sha256 = $sha | .artifacts["wasm32-wasip2"].url = $url' \ + "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" + echo "Patched $manifest with sha256=$sha256 url=$url" fi done done < "$CHECKSUMS" @@ -268,21 +275,41 @@ jobs: for manifest in registry/tools/*.json registry/channels/*.json; do [ -f "$manifest" ] || continue - name=$(jq -r '.name' "$manifest") + # file_stem: JSON filename without extension (e.g. "slack" for slack.json). + # Used for the bundle filename and CI manifest lookup, so patching always + # finds the right file regardless of whether manifest.name matches the filename. + file_stem=$(basename "$manifest" .json) + # ext_name: the manifest's .name field (e.g. "slack-tool"). + # Used for file names *inside* the archive — the installer extracts by manifest.name. + ext_name=$(jq -r '.name' "$manifest") source_dir=$(jq -r '.source.dir' "$manifest") caps_file=$(jq -r '.source.capabilities' "$manifest") crate_name=$(jq -r '.source.crate_name' "$manifest") + ext_version=$(jq -r '.version // ""' "$manifest") if [ ! -d "$source_dir" ]; then - echo "::warning::Source dir '$source_dir' not found for '$name', skipping" + echo "::warning::Source dir '$source_dir' not found for '$file_stem', skipping" continue fi - echo "=== Building $name from $source_dir ===" + # Skip rebuild if this exact version was already built and checksummed. + # Checks that (1) the manifest already has a sha256, and (2) the version + # embedded in the existing artifact URL matches the current manifest version. + # This ensures stable checksums: only rebuild when the source version changes. + existing_sha=$(jq -r '.artifacts["wasm32-wasip2"].sha256 // ""' "$manifest") + existing_url=$(jq -r '.artifacts["wasm32-wasip2"].url // ""' "$manifest") + url_version=$(echo "$existing_url" | sed -n 's/.*-\([0-9].*\)-wasm32-wasip2\.tar\.gz$/\1/p') + + if [[ -n "$ext_version" && "$url_version" == "$ext_version" && -n "$existing_sha" ]]; then + echo "=== Skipping $file_stem v$ext_version — already checksummed at $existing_url ===" + continue + fi + + echo "=== Building $file_stem ($ext_name) v$ext_version from $source_dir ===" # Build WASM component cargo component build --release --manifest-path "$source_dir/Cargo.toml" || { - echo "::warning::Build failed for '$name', skipping" + echo "::warning::Build failed for '$file_stem', skipping" continue } @@ -298,30 +325,36 @@ jobs: done if [ -z "$wasm_path" ]; then - echo "::warning::No WASM output found for '$name', skipping" + echo "::warning::No WASM output found for '$file_stem', skipping" continue fi - # Copy files with standardized names for the archive - cp "$wasm_path" "target/wasm-bundles/${name}.wasm" + # Archive contents use ext_name (manifest .name) — the installer extracts + # files by manifest.name, so these must match even when file_stem differs. + cp "$wasm_path" "target/wasm-bundles/${ext_name}.wasm" caps_path="$source_dir/$caps_file" if [ -f "$caps_path" ]; then - cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json" + cp "$caps_path" "target/wasm-bundles/${ext_name}.capabilities.json" else - echo "::warning::No capabilities file at '$caps_path' for '$name'" + echo "::warning::No capabilities file at '$caps_path' for '$file_stem'" fi - # Create tar.gz bundle - bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz" - (cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi) + # Bundle filename uses file_stem so CI patching can find the manifest by + # filename (e.g. slack-0.1.0-wasm32-wasip2.tar.gz → registry/tools/slack.json). + bundle="target/wasm-bundles/${file_stem}-${ext_version}-wasm32-wasip2.tar.gz" + (cd target/wasm-bundles && if [ -f "${ext_name}.capabilities.json" ]; then + tar czf "${file_stem}-${ext_version}-wasm32-wasip2.tar.gz" "${ext_name}.wasm" "${ext_name}.capabilities.json" + else + tar czf "${file_stem}-${ext_version}-wasm32-wasip2.tar.gz" "${ext_name}.wasm" + fi) # Compute SHA256 sha256=$(sha256sum "$bundle" | cut -d' ' -f1) - echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt + echo "$sha256 ${file_stem}-${ext_version}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt # Clean up intermediate files - rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json" + rm -f "target/wasm-bundles/${ext_name}.wasm" "target/wasm-bundles/${ext_name}.capabilities.json" echo " -> $bundle ($sha256)" done @@ -427,8 +460,10 @@ jobs: with: name: artifacts-wasm-extensions path: target/wasm-bundles/ - - name: Patch manifests with SHA256 + - name: Patch manifests with SHA256 and version-pinned URL shell: bash + env: + RELEASE_TAG: ${{ github.ref_name }} run: | CHECKSUMS="target/wasm-bundles/checksums.txt" if [ ! -f "$CHECKSUMS" ]; then @@ -439,12 +474,17 @@ jobs: while IFS= read -r line; do sha256=$(echo "$line" | awk '{print $1}') filename=$(echo "$line" | awk '{print $2}') - name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//') + # Strip -{version}-wasm32-wasip2.tar.gz to get the extension name. + # Use '.*' (greedy) so pre-release suffixes like -alpha.1 are consumed too. + name=$(echo "$filename" | sed 's/-[0-9].*-wasm32-wasip2\.tar\.gz$//') + url="https://github.com/nearai/ironclaw/releases/download/${RELEASE_TAG}/${filename}" for manifest in registry/tools/${name}.json registry/channels/${name}.json; do if [ -f "$manifest" ]; then - jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" - echo "Patched $manifest with sha256=$sha256" + jq --arg sha "$sha256" --arg url "$url" \ + '.artifacts["wasm32-wasip2"].sha256 = $sha | .artifacts["wasm32-wasip2"].url = $url' \ + "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" + echo "Patched $manifest with sha256=$sha256 url=$url" fi done done < "$CHECKSUMS" @@ -461,8 +501,8 @@ jobs: git commit -m "chore: update WASM artifact SHA256 checksums [skip ci]" git push origin "$BRANCH" gh pr create \ - --title "chore: update WASM artifact SHA256 checksums" \ - --body "Auto-generated by release CI. Updates SHA256 checksums in registry manifests to match the released WASM artifacts." \ + --title "chore: update WASM artifact checksums and version-pinned URLs" \ + --body "Auto-generated by release CI. Updates SHA256 checksums and version-pinned artifact URLs in registry manifests to match the released WASM artifacts. Only extensions whose version changed since the last release are included." \ --base main \ --head "$BRANCH" fi diff --git a/.gitignore b/.gitignore index 80135737..51b461f2 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ trace_*.json # Local Claude Code settings (machine-specific, should not be committed) .claude/settings.local.json +.worktrees/ diff --git a/CLAUDE.md b/CLAUDE.md index 1b454e21..f7c0b403 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,8 @@ src/ │ └── job_manager.rs # Container lifecycle (create, stop, cleanup) │ ├── worker/ # Runs inside Docker containers -│ ├── runtime.rs # Worker execution loop (tool calls, LLM) +│ ├── container.rs # Container worker runtime (ContainerDelegate + shared agentic loop) +│ ├── job.rs # Background job worker (JobDelegate + shared agentic loop) │ ├── claude_bridge.rs # Claude Code bridge (spawns claude CLI) │ └── proxy_llm.rs # LlmProvider that proxies through orchestrator │ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c719811..1c5c6d88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,34 @@ # Contributing +## Getting Started + +```bash +git clone https://github.com/nearai/ironclaw.git +cd ironclaw +./scripts/dev-setup.sh +``` + +This installs the Rust toolchain, WASM targets, git hooks, and runs initial checks. + +## Development Workflow + +```bash +cargo fmt # format +cargo clippy --all --benches --tests --examples --all-features # lint (zero warnings) +cargo test # unit tests +cargo test --features integration # + PostgreSQL tests +``` + +## Code Style + +- Zero clippy warnings policy +- No `.unwrap()` or `.expect()` in production code (tests are fine) +- Use `thiserror` for error types, map errors with context +- Prefer `crate::` for cross-module imports +- Comments for non-obvious logic only + +See `CLAUDE.md` for full style guidelines. + ## Feature Parity Requirement When your change affects a tracked capability, update `FEATURE_PARITY.md` in the same branch. @@ -9,3 +38,23 @@ When your change affects a tracked capability, update `FEATURE_PARITY.md` in the 1. Review the relevant parity rows in `FEATURE_PARITY.md`. 2. Update status/notes if behavior changed. 3. Include the `FEATURE_PARITY.md` diff in your commit when applicable. + +## Review Tracks + +All PRs follow a risk-based review process: + +| Track | Scope | Requirements | +|-------|-------|-------------| +| **A** | Docs, tests, chore, dependency bumps | 1 approval + CI green | +| **B** | Features, refactors, new tools/channels | 1 approval + CI green + test evidence | +| **C** | Security (`src/safety/`, `src/secrets/`), runtime (`src/agent/`, `src/worker/`), database schema, CI workflows | 2 approvals + rollback plan documented | + +Select the appropriate track in the PR template based on what your changes touch. + +## Database Changes + +IronClaw uses dual-backend persistence (PostgreSQL + libSQL). All new persistence features must support both backends. See `src/db/CLAUDE.md`. + +## Adding Dependencies + +Run `cargo deny check` before adding new dependencies to verify license compatibility and check for known advisories. diff --git a/COVERAGE_PLAN.md b/COVERAGE_PLAN.md index c9d7d73b..af5f872c 100644 --- a/COVERAGE_PLAN.md +++ b/COVERAGE_PLAN.md @@ -63,12 +63,12 @@ These files account for the vast majority of the coverage gap: | `src/main.rs` | 740 | 522 | 29.4% | 485 | | `src/channels/web/handlers/jobs.rs` | 513 | 456 | 11.1% | 430 | | `src/tools/builder/core.rs` | 524 | 456 | 13.0% | 429 | -| `src/agent/worker.rs` | 1,078 | 467 | 56.7% | 413 | +| `src/worker/job.rs` | 1,078 | 467 | 56.7% | 413 | | `src/channels/web/handlers/chat.rs` | 564 | 417 | 26.1% | 388 | | `src/tools/wasm/wrapper.rs` | 1,005 | 436 | 56.6% | 385 | | `src/channels/signal.rs` | 1,814 | 472 | 74.0% | 381 | | `src/tools/mcp/auth.rs` | 472 | 378 | 19.9% | 354 | -| `src/worker/runtime.rs` | 350 | 330 | 5.7% | 312 | +| `src/worker/container.rs` | 350 | 330 | 5.7% | 312 | | `src/tools/builtin/job.rs` | 1,014 | 359 | 64.6% | 308 | | `src/cli/mcp.rs` | 322 | 319 | 0.9% | 302 | | `src/cli/oauth_defaults.rs` | 730 | 335 | 54.1% | 298 | @@ -346,7 +346,7 @@ Test slash commands through the agent loop. ### Trace: Worker Multi-Turn Execution -**Covers:** `agent/worker.rs` (+413 lines), `agent/agent_loop.rs` (+207 lines) +**Covers:** `worker/job.rs` (+413 lines), `agent/agent_loop.rs` (+207 lines) Test multi-turn tool calling, error recovery, and completion flows. @@ -769,7 +769,7 @@ HTTP proxy for container network access. - `test_proxy_connect_tunnel` -- HTTPS CONNECT method handling - `test_proxy_logging` -- request/response logging -### `src/worker/runtime.rs` -- 5.7% -> 95% (+312 lines) +### `src/worker/container.rs` -- 5.7% -> 95% (+312 lines) Worker execution loop (runs inside containers). diff --git a/Cargo.toml b/Cargo.toml index b3551b44..8f5bc29a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ exclude = [ "tools-src/google-slides", "tools-src/slack", "tools-src/telegram", + "fuzz", ] [package] @@ -214,10 +215,14 @@ bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types name = "html_to_markdown" required-features = ["html-to-markdown"] +[profile.release] +strip = true # Remove debug symbols from release binaries + # The profile that 'cargo dist' will build with [profile.dist] inherits = "release" -lto = "thin" +lto = "fat" # Full cross-crate LTO (slow build, better codegen) +codegen-units = 1 # Single codegen unit for maximum optimization # Config for 'dist' [workspace.metadata.dist] diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index d6952cbd..634131fc 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -46,14 +46,14 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Bonjour/mDNS discovery | ✅ | ❌ | | | Tailscale integration | ✅ | ❌ | | | Health check endpoints | ✅ | ✅ | /api/health + /api/gateway/status + /healthz + /readyz, with channel-backed readiness probes | -| `doctor` diagnostics | ✅ | ❌ | | +| `doctor` diagnostics | ✅ | 🚧 | 16 checks: settings, LLM, DB, embeddings, routines, gateway, MCP, skills, secrets, service, Docker daemon, tunnel binaries | | Agent event broadcast | ✅ | 🚧 | SSE broadcast manager exists (SseManager) but tool/job-state events not fully wired | | Channel health monitor | ✅ | ❌ | Auto-restart with configurable interval | | Presence system | ✅ | ❌ | Beacons on connect, system presence for agents | | Trusted-proxy auth mode | ✅ | ❌ | Header-based auth for reverse proxies | | APNs push pipeline | ✅ | ❌ | Wake disconnected iOS nodes via push | | Oversized payload guard | ✅ | 🚧 | HTTP webhook has 64KB body limit + Content-Length check; no chat.history cap | -| Pre-prompt context diagnostics | ✅ | ❌ | Context size logging before prompt | +| Pre-prompt context diagnostics | ✅ | 🚧 | Token breakdown logged before LLM call (conversational dispatcher path); other LLM entry points not yet covered | ### Owner: _Unassigned_ @@ -175,7 +175,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | `message send` | ✅ | ❌ | P2 | Send to channels | | `browser` | ✅ | ❌ | P3 | Browser automation | | `sandbox` | ✅ | ✅ | - | WASM sandbox | -| `doctor` | ✅ | ❌ | P2 | Diagnostics | +| `doctor` | ✅ | 🚧 | P2 | 16 subsystem checks | | `logs` | ✅ | ❌ | P3 | Query logs | | `update` | ✅ | ❌ | P3 | Self-update | | `completion` | ✅ | ✅ | - | Shell completion | diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 00000000..d6865a24 --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "ironclaw-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +serde_json = "1" + +[dependencies.ironclaw] +path = ".." + +[[bin]] +name = "fuzz_safety_sanitizer" +path = "fuzz_targets/fuzz_safety_sanitizer.rs" +doc = false + +[[bin]] +name = "fuzz_safety_validator" +path = "fuzz_targets/fuzz_safety_validator.rs" +doc = false + +[[bin]] +name = "fuzz_leak_detector" +path = "fuzz_targets/fuzz_leak_detector.rs" +doc = false + +[[bin]] +name = "fuzz_tool_params" +path = "fuzz_targets/fuzz_tool_params.rs" +doc = false + +[[bin]] +name = "fuzz_config_env" +path = "fuzz_targets/fuzz_config_env.rs" +doc = false diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 00000000..c4c27c69 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,43 @@ +# IronClaw Fuzz Targets + +Fuzz testing for security-critical input parsing paths using [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz) (libFuzzer). + +## Targets + +| Target | What it exercises | +|--------|-------------------| +| `fuzz_safety_sanitizer` | Prompt injection pattern detection (Aho-Corasick + regex) | +| `fuzz_safety_validator` | Input validation (length, encoding, forbidden patterns) | +| `fuzz_leak_detector` | Secret leak detection (API keys, tokens, credentials) | +| `fuzz_tool_params` | Tool parameter and schema JSON validation | +| `fuzz_config_env` | SafetyLayer end-to-end (sanitize, validate, policy check) | + +## Setup + +```bash +cargo install cargo-fuzz +rustup install nightly +``` + +## Running + +```bash +# Run a specific target (runs until stopped or crash found) +cargo +nightly fuzz run fuzz_safety_sanitizer + +# Run with a time limit (5 minutes) +cargo +nightly fuzz run fuzz_leak_detector -- -max_total_time=300 + +# Run all targets for 60 seconds each +for target in fuzz_safety_sanitizer fuzz_safety_validator fuzz_leak_detector fuzz_tool_params fuzz_config_env; do + echo "==> $target" + cargo +nightly fuzz run "$target" -- -max_total_time=60 +done +``` + +## Adding New Targets + +1. Create `fuzz/fuzz_targets/fuzz_.rs` following the existing pattern +2. Add a `[[bin]]` entry in `fuzz/Cargo.toml` +3. Create `fuzz/corpus/fuzz_/` for seed inputs +4. Exercise real IronClaw code paths, not just generic serde diff --git a/fuzz/corpus/fuzz_config_env/.gitkeep b/fuzz/corpus/fuzz_config_env/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fuzz/corpus/fuzz_leak_detector/.gitkeep b/fuzz/corpus/fuzz_leak_detector/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fuzz/corpus/fuzz_safety_sanitizer/.gitkeep b/fuzz/corpus/fuzz_safety_sanitizer/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fuzz/corpus/fuzz_safety_validator/.gitkeep b/fuzz/corpus/fuzz_safety_validator/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fuzz/corpus/fuzz_tool_params/.gitkeep b/fuzz/corpus/fuzz_tool_params/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/fuzz/fuzz_targets/fuzz_config_env.rs b/fuzz/fuzz_targets/fuzz_config_env.rs new file mode 100644 index 00000000..265a85e9 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_config_env.rs @@ -0,0 +1,55 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; + +use ironclaw::safety::{LeakDetector, Sanitizer, Validator}; + +fuzz_target!(|data: &[u8]| { + if let Ok(input) = std::str::from_utf8(data) { + // Exercise Sanitizer: detect and neutralize prompt injection attempts. + let sanitizer = Sanitizer::new(); + let sanitized = sanitizer.sanitize(input); + // The sanitized content must never be empty when input is non-empty, + // because sanitization wraps/escapes rather than deleting. + if !input.is_empty() { + assert!( + !sanitized.content.is_empty(), + "sanitize() produced empty content for non-empty input" + ); + } + // If no modification occurred, content must equal input. + if !sanitized.was_modified { + assert_eq!(sanitized.content, input); + } + + // Exercise Validator: input validation (length, encoding, patterns). + let validator = Validator::new(); + let result = validator.validate(input); + // ValidationResult must always be well-formed: if valid, no errors. + if result.is_valid { + assert!( + result.errors.is_empty(), + "valid result should have no errors" + ); + } + + // Exercise LeakDetector: secret detection (API keys, tokens, etc.). + let detector = LeakDetector::new(); + let scan = detector.scan(input); + // scan_and_clean must not panic and must return valid UTF-8. + let cleaned = detector.scan_and_clean(input); + if let Ok(ref clean_str) = cleaned { + // Cleaned output must never be longer than original + redaction markers. + // At minimum it should be valid UTF-8 (guaranteed by String type). + let _ = clean_str.len(); + } + // If scan found no matches, scan_and_clean should return the input unchanged. + if scan.matches.is_empty() { + if let Ok(ref clean_str) = cleaned { + assert_eq!( + clean_str, input, + "scan_and_clean changed content despite no matches" + ); + } + } + } +}); diff --git a/fuzz/fuzz_targets/fuzz_leak_detector.rs b/fuzz/fuzz_targets/fuzz_leak_detector.rs new file mode 100644 index 00000000..f1e6e09c --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_leak_detector.rs @@ -0,0 +1,23 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use ironclaw::safety::LeakDetector; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let detector = LeakDetector::new(); + + // Exercise scan path + let result = detector.scan(s); + // Invariant: if should_block, there must be matches + if result.should_block { + assert!(!result.matches.is_empty()); + } + // Invariant: match locations must be valid + for m in &result.matches { + assert!(m.location.end <= s.len()); + } + + // Exercise scan_and_clean path + let _ = detector.scan_and_clean(s); + } +}); diff --git a/fuzz/fuzz_targets/fuzz_safety_sanitizer.rs b/fuzz/fuzz_targets/fuzz_safety_sanitizer.rs new file mode 100644 index 00000000..32db887d --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_safety_sanitizer.rs @@ -0,0 +1,23 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use ironclaw::safety::Sanitizer; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let sanitizer = Sanitizer::new(); + + // Exercise the main sanitization path + let result = sanitizer.sanitize(s); + // Verify invariant: warnings should have valid ranges + for w in &result.warnings { + assert!(w.location.end <= s.len()); + } + // Verify invariant: critical severity triggers modification + let has_critical = result.warnings.iter().any(|w| { + w.severity == ironclaw::safety::Severity::Critical + }); + if has_critical { + assert!(result.was_modified); + } + } +}); diff --git a/fuzz/fuzz_targets/fuzz_safety_validator.rs b/fuzz/fuzz_targets/fuzz_safety_validator.rs new file mode 100644 index 00000000..065bc86d --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_safety_validator.rs @@ -0,0 +1,21 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use ironclaw::safety::Validator; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + let validator = Validator::new(); + + // Exercise input validation + let result = validator.validate(s); + // Invariant: empty input is always invalid + if s.is_empty() { + assert!(!result.is_valid); + } + + // Exercise tool parameter validation with arbitrary JSON + if let Ok(value) = serde_json::from_str::(s) { + let _ = validator.validate_tool_params(&value); + } + } +}); diff --git a/fuzz/fuzz_targets/fuzz_tool_params.rs b/fuzz/fuzz_targets/fuzz_tool_params.rs new file mode 100644 index 00000000..52e39867 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_tool_params.rs @@ -0,0 +1,22 @@ +#![no_main] +use libfuzzer_sys::fuzz_target; +use ironclaw::safety::Validator; +use ironclaw::tools::validate_tool_schema; + +fuzz_target!(|data: &[u8]| { + if let Ok(s) = std::str::from_utf8(data) { + // Try parsing as JSON and validating as tool parameters + if let Ok(value) = serde_json::from_str::(s) { + // Exercise Validator::validate_tool_params with arbitrary JSON + let validator = Validator::new(); + let result = validator.validate_tool_params(&value); + // Invariant: result should always be well-formed + if !result.is_valid { + assert!(!result.errors.is_empty()); + } + + // Exercise validate_tool_schema with arbitrary JSON as a schema + let _ = validate_tool_schema(&value, "fuzz"); + } + } +}); diff --git a/migrations/V12__job_token_budget.sql b/migrations/V12__job_token_budget.sql new file mode 100644 index 00000000..fbda73e3 --- /dev/null +++ b/migrations/V12__job_token_budget.sql @@ -0,0 +1,7 @@ +-- Add token budget tracking columns to agent_jobs. +-- +-- Tracks max_tokens (configured limit per job) and total_tokens_used (running total) +-- to enforce job-level token budgets and prevent budget bypass via user-supplied metadata. + +ALTER TABLE agent_jobs ADD COLUMN max_tokens BIGINT NOT NULL DEFAULT 0; +ALTER TABLE agent_jobs ADD COLUMN total_tokens_used BIGINT NOT NULL DEFAULT 0; diff --git a/registry/channels/discord.json b/registry/channels/discord.json index 1351c96d..1b13658a 100644 --- a/registry/channels/discord.json +++ b/registry/channels/discord.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz", - "sha256": "85b424604482da3fb9badb56a0360ff4c93670bc7be0ad7f57ef9d85ff972b6f" + "sha256": null } }, "auth_summary": { diff --git a/registry/channels/slack.json b/registry/channels/slack.json index 218fc12a..593c2758 100644 --- a/registry/channels/slack.json +++ b/registry/channels/slack.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz", - "sha256": "9190b8250bd20c22a8c97b1ea19a6590624a69d6c63a5f5c240a7840a4966286" + "sha256": null } }, "auth_summary": { diff --git a/registry/channels/telegram.json b/registry/channels/telegram.json index 965c469d..07975121 100644 --- a/registry/channels/telegram.json +++ b/registry/channels/telegram.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz", - "sha256": "55f2a56e7afd129a48fd49b019f12f9638705defa53fa323ad3b8978d7c59664" + "sha256": null } }, "auth_summary": { diff --git a/registry/channels/whatsapp.json b/registry/channels/whatsapp.json index 098f51d7..5e7c2bc3 100644 --- a/registry/channels/whatsapp.json +++ b/registry/channels/whatsapp.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz", - "sha256": "06bcf315df93af9f683134f4055eb810c602863d8c4a632e3733a10217cc5a89" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/github.json b/registry/tools/github.json index 2ba222ba..bf7af291 100644 --- a/registry/tools/github.json +++ b/registry/tools/github.json @@ -20,7 +20,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz", - "sha256": "c443328a3f10b6a4cf4d3d62c9217aca204f6467ef753d986b58ca966ca53514" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/gmail.json b/registry/tools/gmail.json index 9ef1a0f9..2bdf6350 100644 --- a/registry/tools/gmail.json +++ b/registry/tools/gmail.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz", - "sha256": "e4f0095890d22e3de8e9d516f2e1e91964f8ff4acdaaa19f0a7094a1f2d7786b" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/google-calendar.json b/registry/tools/google-calendar.json index 40342b48..7b0afd80 100644 --- a/registry/tools/google-calendar.json +++ b/registry/tools/google-calendar.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz", - "sha256": "2d202bd838de94677c91ea6473c7155f021c0500cf91794d17639b1b27446b3d" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/google-docs.json b/registry/tools/google-docs.json index 9aaf9d17..b564d0e6 100644 --- a/registry/tools/google-docs.json +++ b/registry/tools/google-docs.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz", - "sha256": "7a5e40fe58199e34f7625e11d22e5601cdfd2a94a10193a83f1925180bbb66df" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/google-drive.json b/registry/tools/google-drive.json index 74bc825c..180aaa1e 100644 --- a/registry/tools/google-drive.json +++ b/registry/tools/google-drive.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz", - "sha256": "d19f856fde0ae0320fd3f636a34116af1df0b59698c3684b686e8412a60e887f" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/google-sheets.json b/registry/tools/google-sheets.json index 716880db..82575182 100644 --- a/registry/tools/google-sheets.json +++ b/registry/tools/google-sheets.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz", - "sha256": "e113c317f9fa21ea68d0ec8accbba4a62a8222ff3c4655ae85e1e58e01de3250" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/google-slides.json b/registry/tools/google-slides.json index 25e9a64a..5127b17d 100644 --- a/registry/tools/google-slides.json +++ b/registry/tools/google-slides.json @@ -18,7 +18,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz", - "sha256": "7875a5ae1283e57937e0618bf14465f4bb4ee7f49110312382670202f4c567a5" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/slack.json b/registry/tools/slack.json index e9f7e6d2..fe038438 100644 --- a/registry/tools/slack.json +++ b/registry/tools/slack.json @@ -18,7 +18,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-tool-wasm32-wasip2.tar.gz", - "sha256": "9190b8250bd20c22a8c97b1ea19a6590624a69d6c63a5f5c240a7840a4966286" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/telegram.json b/registry/tools/telegram.json index a01b1961..ab036396 100644 --- a/registry/tools/telegram.json +++ b/registry/tools/telegram.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-mtproto-wasm32-wasip2.tar.gz", - "sha256": "55f2a56e7afd129a48fd49b019f12f9638705defa53fa323ad3b8978d7c59664" + "sha256": null } }, "auth_summary": { diff --git a/registry/tools/web-search.json b/registry/tools/web-search.json index e61ab0dd..9c9111ac 100644 --- a/registry/tools/web-search.json +++ b/registry/tools/web-search.json @@ -19,7 +19,7 @@ "artifacts": { "wasm32-wasip2": { "url": "https://github.com/nearai/ironclaw/releases/latest/download/web-search-wasm32-wasip2.tar.gz", - "sha256": "dd7e54956ee0b3037ca3506dbcbd20efcc4cd2749175ed511b3640b09f77506a" + "sha256": null } }, "auth_summary": { diff --git a/skills/ironclaw-workflow-orchestrator/SKILL.md b/skills/ironclaw-workflow-orchestrator/SKILL.md new file mode 100644 index 00000000..88d01441 --- /dev/null +++ b/skills/ironclaw-workflow-orchestrator/SKILL.md @@ -0,0 +1,80 @@ +--- +name: ironclaw-workflow-orchestrator +description: "Install and operate a full GitHub issue-to-merge workflow in IronClaw using event-driven and cron routines. Use when setting up or tuning autonomous project orchestration: issue intake, planning, maintainer feedback handling, branch/PR execution, CI/comment follow-up, batched staging review every 8 hours, and memory updates from merge outcomes." +--- + +# IronClaw Workflow Orchestrator + +## Overview +Use this skill to install and maintain a complete project workflow as routines, not core code changes. It maps GitHub webhook events plus scheduled checks into plan/update/implement/review/merge loops with explicit staging-batch analysis. + +## Workflow +1. Gather workflow parameters. +2. Verify runtime prerequisites. +3. Install or update routine set from templates. +4. Run a dry test with `event_emit`. +5. Monitor outcomes and tune prompts/filters. + +## Parameters +Collect these values before creating routines: +- `repository`: `owner/repo` (required) +- `maintainers`: GitHub handles allowed to trigger implement/replan actions +- `staging_branch`: default `staging` +- `main_branch`: default `main` +- `batch_interval_hours`: default `8` +- `implementation_label`: default `autonomous-impl` + +## Prerequisites +Before installing routines, verify: +- Routines system enabled. +- GitHub tool authenticated (for issue/PR/comment/status operations). +- Events are emitted via `event_emit` tool calls (a future HTTP webhook ingestion endpoint is planned but not yet available). + +## Install Procedure +1. Open [`workflow-routines.md`](references/workflow-routines.md). +2. For each template block: +- replace placeholders (`{{repository}}`, `{{maintainers}}`, branch names) +- call `routine_create` +3. If a routine already exists: +- use `routine_update` instead of creating duplicates +- keep names stable so long-lived metrics/history stay intact +4. Confirm install with `routine_list` and `routine_history`. + +## Routine Set +Install these routines: +- `wf-issue-plan`: on `issue.opened` or `issue.reopened`, generate implementation plan comment/checklist. +- `wf-maintainer-comment-gate`: on maintainer comments, decide update-plan vs start implementation. +- `wf-pr-monitor-loop`: on PR open/sync/review-comment/review, address feedback and refresh branch. +- `wf-ci-fix-loop`: on CI status/check failures, apply fixes and push updates. +- `wf-staging-batch-review`: every 8h, review ready PRs, merge into staging, run deep batch correctness analysis, fix findings, then merge staging -> main. +- `wf-learning-memory`: on merged PRs, extract mistakes/lessons and write to shared memory. + +## Event Filters +Prefer top-level filters for stability: +- `repository` (string) +- `sender` (string) +- `issue_number` / `pr_number` +- `ci_status`, `ci_conclusion` +- `review_state`, `comment_author` + +Use narrow filters to avoid accidental triggers across repos. + +## Operating Rules +- All implementation work must occur on non-main branches. +- PR loop must resolve both human and AI review comments. +- On conflicts with `origin/main`, refresh branch before continuing. +- Staging-batch routine is the only path for bulk correctness verification before mainline merge. +- Memory update routine runs only after successful merge. + +## Validation +After install, run: +1. `event_emit` with a synthetic `issue.opened` payload for the target repo. +2. Confirm at least one routine fired. +3. Check corresponding `routine_history` entries. +4. Confirm no unrelated routines fired. + +## When To Update Templates +Update this skill when: +- GitHub event names/payload fields change. +- Team review policy changes (e.g., staging cadence, maintainer gates). +- New CI policy requires different failure routing. diff --git a/skills/ironclaw-workflow-orchestrator/agents/openai.yaml b/skills/ironclaw-workflow-orchestrator/agents/openai.yaml new file mode 100644 index 00000000..3febe0ff --- /dev/null +++ b/skills/ironclaw-workflow-orchestrator/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "IronClaw Workflow Orchestrator" + short_description: "Install and run event-driven GitHub workflow routines" + default_prompt: "Set up the full issue-to-merge workflow using routines and event triggers." diff --git a/skills/ironclaw-workflow-orchestrator/references/workflow-routines.md b/skills/ironclaw-workflow-orchestrator/references/workflow-routines.md new file mode 100644 index 00000000..74a5fb92 --- /dev/null +++ b/skills/ironclaw-workflow-orchestrator/references/workflow-routines.md @@ -0,0 +1,128 @@ +# Workflow Routine Templates + +Replace `{{...}}` placeholders before use. + +## 1) Issue -> Plan + +```json +{ + "name": "wf-issue-plan", + "description": "Create implementation plan when a new issue arrives", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "issue.opened", + "event_filters": { + "repository": "{{repository}}" + }, + "action_type": "full_job", + "prompt": "For issue #{{issue_number}} in {{repository}}, produce a concrete implementation plan with milestones, edge cases, and tests. Post/update an issue comment with the plan.", + "cooldown_secs": 30 +} +``` + +## 2) Maintainer Comment Gate (Update Plan vs Implement) + +Trigger per-maintainer by creating one routine per handle, or maintain a shared author convention. + +```json +{ + "name": "wf-maintainer-comment-gate-{{maintainer}}", + "description": "React to maintainer guidance comments on issues/PRs", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "pr.comment.created", + "event_filters": { + "repository": "{{repository}}", + "comment_author": "{{maintainer}}" + }, + "action_type": "full_job", + "prompt": "Read the maintainer comment and decide: update plan or start/continue implementation. If plan changes are requested, edit the plan artifact first. If implementation is requested, continue on the feature branch and update PR status/comment.", + "cooldown_secs": 20 +} +``` + +## 3) PR Monitor Loop + +```json +{ + "name": "wf-pr-monitor-loop", + "description": "Keep PR healthy: address review comments and refresh branch", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "pr.synchronize", + "event_filters": { + "repository": "{{repository}}" + }, + "action_type": "full_job", + "prompt": "For PR #{{pr_number}}, collect open review comments and unresolved threads, apply fixes, push branch updates, and summarize remaining blockers. If conflict with {{main_branch}}, rebase/merge from origin/{{main_branch}} and resolve safely.", + "cooldown_secs": 20 +} +``` + +## 4) CI Failure Fix Loop + +```json +{ + "name": "wf-ci-fix-loop", + "description": "Fix failing CI checks on active PRs", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "ci.check_run.completed", + "event_filters": { + "repository": "{{repository}}", + "ci_conclusion": "failure" + }, + "action_type": "full_job", + "prompt": "Find failing check details for PR #{{pr_number}}, implement minimal safe fixes, rerun or await CI, and post concise status updates. Prioritize deterministic and test-backed fixes.", + "cooldown_secs": 20 +} +``` + +## 5) Staging Batch Review (Every 8h) + +```json +{ + "name": "wf-staging-batch-review", + "description": "Batch correctness review through staging, then merge to main", + "trigger_type": "cron", + "schedule": "0 0 */{{batch_interval_hours}} * * *", + "action_type": "full_job", + "prompt": "Every cycle: list ready PRs, merge ready ones into {{staging_branch}}, run deep correctness analysis in batch, fix discovered issues on affected branches, ensure CI green, then merge {{staging_branch}} into {{main_branch}} if clean.", + "cooldown_secs": 120 +} +``` + +## 6) Post-Merge Learning -> Common Memory + +```json +{ + "name": "wf-learning-memory", + "description": "Capture merge learnings into shared memory", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "pr.closed", + "event_filters": { + "repository": "{{repository}}", + "pr_merged": "true" + }, + "action_type": "full_job", + "prompt": "From merged PR #{{pr_number}}, extract preventable mistakes, reviewer themes, CI failure causes, and successful patterns. Write/update a shared memory doc with actionable rules to reduce cycle time and regressions.", + "cooldown_secs": 30 +} +``` + +## Optional: Synthetic Event Test + +```json +{ + "source": "github", + "event_type": "issue.opened", + "payload": { + "repository": "{{repository}}", + "issue_number": 99999, + "sender": "test-bot" + } +} +``` + +Use with `event_emit` after routine install. diff --git a/src/agent/CLAUDE.md b/src/agent/CLAUDE.md index 40221341..e55c9591 100644 --- a/src/agent/CLAUDE.md +++ b/src/agent/CLAUDE.md @@ -14,14 +14,15 @@ Core agent logic. This is the most complex subsystem — read this before workin | `session_manager.rs` | Lifecycle: create/lookup sessions, map external thread IDs to internal UUIDs, prune stale sessions, manage undo managers. | | `router.rs` | Routes explicit `/commands` to `MessageIntent`. Natural language bypasses the router entirely. | | `scheduler.rs` | Parallel job scheduling. Maintains `jobs` map (full LLM-driven) and `subtasks` map (tool-exec/background). | -| `worker.rs` | Per-job execution for background scheduler jobs: calls LLM, runs tools, handles the reasoning loop. Distinct from `dispatcher.rs`. | +| *(moved to `src/worker/job.rs`)* | Per-job execution now lives in `src/worker/job.rs` as `JobDelegate`, using the shared `run_agentic_loop()` engine. | +| `agentic_loop.rs` | Shared agentic loop engine: `run_agentic_loop()`, `LoopDelegate` trait, `LoopOutcome`, `LoopSignal`, `TextAction`. All three execution paths (chat, job, container) delegate to this. | | `compaction.rs` | Context window management: summarize old turns, write to workspace daily log, trim context. Three strategies. | | `context_monitor.rs` | Detects memory pressure. Suggests `CompactionStrategy` based on usage level. | | `self_repair.rs` | Detects stuck jobs and broken tools, attempts recovery. | | `heartbeat.rs` | Proactive periodic execution. Reads `HEARTBEAT.md`, notifies via channel if findings. | | `submission.rs` | Parses all user submissions into typed variants before routing. | | `undo.rs` | Turn-based undo/redo with checkpoints. Checkpoints store message lists (max 20 by default). | -| `routine.rs` | `Routine` types: `Trigger` (cron/event/webhook/manual) + `RoutineAction` (lightweight/full_job) + `RoutineGuardrails`. | +| `routine.rs` | `Routine` types: `Trigger` (cron/event/system_event/manual) + `RoutineAction` (lightweight/full_job) + `RoutineGuardrails`. | | `routine_engine.rs` | Cron ticker and event matcher. Fires routines when triggers match. Lightweight runs inline; full_job dispatches to `Scheduler`. | | `task.rs` | Task types for the scheduler: `Job`, `ToolExec`, `Background`. Used by `spawn_subtask` and `spawn_batch`. | | `cost_guard.rs` | LLM spend and action-rate enforcement. Tracks daily budget (cents) and hourly call rate. Lives in `AgentDeps`. | @@ -49,26 +50,28 @@ Session (per user) ## Agentic Loop (dispatcher.rs) -The `dispatcher.rs` module handles **direct conversational turns** (user messages processed inline by the main agent). Background scheduler jobs use `worker.rs` instead — these are two separate execution paths. +All three execution paths (chat, job, container) now use the shared `run_agentic_loop()` engine in `agentic_loop.rs`, each providing their own `LoopDelegate` implementation: + +- **`ChatDelegate`** (`dispatcher.rs`) — conversational turns, tool approval, skill context injection +- **`JobDelegate`** (`src/worker/job.rs`) — background scheduler jobs, planning support, completion detection +- **`ContainerDelegate`** (`src/worker/container.rs`) — Docker container worker, sequential tool exec, HTTP event streaming ``` -run_agentic_loop() [dispatcher.rs — conversational turns] - 1. Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.) - 2. Detect group chat from metadata; exclude MEMORY.md if group chat - 3. Select active skills (keyword/pattern scoring against message content) - 4. Build skill context block (injected before user message) - 5. LLM call → text response OR tool calls - 6. If tool calls: - a. Check tool approval (session auto-approvals, pending approval queue) - b. Execute tools (parallel via JoinSet) - c. Sanitize results through SafetyLayer - d. Feed results back → goto 5 - 7. Return AgenticLoopResult::Response or NeedApproval +run_agentic_loop(delegate, reasoning, reason_ctx, config) + 1. Check signals (stop/cancel) via delegate.check_signals() + 2. Pre-LLM hook via delegate.before_llm_call() + 3. LLM call via delegate.call_llm() + 4. If text response → delegate.handle_text_response() → Continue or Return + 5. If tool calls → delegate.execute_tool_calls() → Continue or Return + 6. Post-iteration hook via delegate.after_iteration() + 7. Repeat until LoopOutcome returned or max_iterations reached ``` -**Tool approval:** Tools flagged `requires_approval` pause the loop and return `NeedApproval`. The web gateway stores the `PendingApproval` in session state and sends an `approval_needed` SSE event. The user's approval/deny resumes the loop. +**Tool approval:** Tools flagged `requires_approval` pause the loop — `ChatDelegate` returns `LoopOutcome::NeedApproval(pending)`. The web gateway stores the `PendingApproval` in session state and sends an `approval_needed` SSE event. The user's approval/deny resumes the loop. -**worker.rs vs dispatcher.rs:** `dispatcher.rs` runs the agentic loop for user-initiated conversational turns (holds session lock, tracks turns). `worker.rs` is spawned by the `Scheduler` for background jobs created via `CreateJob` / `/job` — it runs independently of the session and has its own LLM reasoning loop with planning support (`use_planning` flag). +**Shared tool execution:** `tools/execute.rs` provides `execute_tool_with_safety()` (validate → timeout → execute → serialize) and `process_tool_result()` (sanitize → wrap → ChatMessage), used by all three delegates. + +**ChatDelegate vs JobDelegate:** `ChatDelegate` runs for user-initiated conversational turns (holds session lock, tracks turns). `JobDelegate` is spawned by the `Scheduler` for background jobs created via `CreateJob` / `/job` — it runs independently of the session and has planning support (`use_planning` flag). ## Command Routing (router.rs) diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 15853f14..d95f3e46 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -738,6 +738,18 @@ impl Agent { } async fn handle_message(&self, message: &IncomingMessage) -> Result, Error> { + // Log at info level only for tracking without exposing PII (user_id can be a phone number) + tracing::info!(message_id = %message.id, "Processing message"); + + // Log sensitive details at debug level for troubleshooting + tracing::debug!( + message_id = %message.id, + user_id = %message.user_id, + channel = %message.channel, + thread_id = ?message.thread_id, + "Message details" + ); + // Set message tool context for this turn (current channel and target) // For Signal, use signal_target from metadata (group:ID or phone number), // otherwise fall back to user_id @@ -753,7 +765,7 @@ impl Agent { // Parse submission type first let mut submission = SubmissionParser::parse(&message.content); - tracing::debug!( + tracing::trace!( "[agent_loop] Parsed submission: {:?}", std::any::type_name_of_val(&submission) ); @@ -786,10 +798,19 @@ impl Agent { // Hydrate thread from DB if it's a historical thread not in memory if let Some(ref external_thread_id) = message.thread_id { + tracing::trace!( + message_id = %message.id, + thread_id = %external_thread_id, + "Hydrating thread from DB" + ); self.maybe_hydrate_thread(message, external_thread_id).await; } // Resolve session and thread + tracing::debug!( + message_id = %message.id, + "Resolving session and thread" + ); let (session, thread_id) = self .session_manager .resolve_thread( @@ -798,6 +819,11 @@ impl Agent { message.thread_id.as_deref(), ) .await; + tracing::debug!( + message_id = %message.id, + thread_id = %thread_id, + "Resolved session and thread" + ); // Auth mode interception: if the thread is awaiting a token, route // the message directly to the credential store. Nothing touches @@ -827,7 +853,7 @@ impl Agent { } } - tracing::debug!( + tracing::trace!( "Received message from {} on {} ({} chars)", message.user_id, message.channel, diff --git a/src/agent/agentic_loop.rs b/src/agent/agentic_loop.rs new file mode 100644 index 00000000..0e5bef9d --- /dev/null +++ b/src/agent/agentic_loop.rs @@ -0,0 +1,587 @@ +//! Unified agentic loop engine. +//! +//! Provides a single implementation of the core LLM call → tool execution → +//! result processing → context update → repeat cycle. Three consumers +//! (chat dispatcher, job worker, container runtime) customize behavior +//! via the `LoopDelegate` trait. + +use async_trait::async_trait; + +use crate::agent::session::PendingApproval; +use crate::error::Error; +use crate::llm::{ChatMessage, Reasoning, ReasoningContext, RespondResult}; + +/// Signal from the delegate indicating how the loop should proceed. +pub enum LoopSignal { + /// Continue normally. + Continue, + /// Stop the loop gracefully. + Stop, + /// Inject a user message into context and continue. + InjectMessage(String), +} + +/// Outcome of a text response from the LLM. +pub enum TextAction { + /// Return this as the final loop result. + Return(LoopOutcome), + /// Continue the loop (text was handled but loop should proceed). + Continue, +} + +/// Final outcome of the agentic loop. +pub enum LoopOutcome { + /// Completed with a text response. + Response(String), + /// Loop was stopped by a signal. + Stopped, + /// Max iterations exceeded. + MaxIterations, + /// A tool requires user approval before continuing (chat delegate only). + NeedApproval(Box), +} + +/// Configuration for the agentic loop. +pub struct AgenticLoopConfig { + pub max_iterations: usize, + pub enable_tool_intent_nudge: bool, + pub max_tool_intent_nudges: u32, +} + +impl Default for AgenticLoopConfig { + fn default() -> Self { + Self { + max_iterations: 50, + enable_tool_intent_nudge: true, + max_tool_intent_nudges: 2, + } + } +} + +/// Strategy trait — each consumer implements this to customize I/O and lifecycle. +/// +/// The shared loop calls these methods at well-defined points. Consumers +/// implement only the behavior that differs between chat, job, and container +/// contexts. The loop itself handles the common logic: tool intent nudge, +/// iteration counting, tool definition refresh, and the respond → execute → process cycle. +/// +/// # `Send + Sync` requirement +/// +/// This trait requires `Send + Sync` because the loop accepts `&dyn LoopDelegate`. +/// Delegates using borrowed references (e.g. `ChatDelegate<'a>`) must ensure all +/// borrowed fields are `Send + Sync`. This is a load-bearing constraint: if a +/// delegate needs to be spawned into a detached task, it must use `Arc`-based +/// ownership instead of borrows (as `JobDelegate` and `ContainerDelegate` do). +#[async_trait] +pub trait LoopDelegate: Send + Sync { + /// Called at the start of each iteration. Check for external signals + /// (cancellation, user messages, stop requests). + async fn check_signals(&self) -> LoopSignal; + + /// Called before the LLM call. Allows the delegate to refresh tool + /// definitions, enforce cost guards, or inject messages. + /// Return `Some(outcome)` to break the loop early. + async fn before_llm_call( + &self, + reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Option; + + /// Call the LLM and return the result. Delegates own the LLM call + /// to handle consumer-specific concerns (rate limiting, auto-compaction, + /// cost tracking, force_text mode). + async fn call_llm( + &self, + reasoning: &Reasoning, + reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Result; + + /// Handle a text-only response from the LLM. + /// Return `TextAction::Return` to exit the loop, `TextAction::Continue` to proceed. + async fn handle_text_response( + &self, + text: &str, + reason_ctx: &mut ReasoningContext, + ) -> TextAction; + + /// Execute tool calls and add results to context. + /// Return `Some(outcome)` to break the loop (e.g. approval needed). + async fn execute_tool_calls( + &self, + tool_calls: Vec, + content: Option, + reason_ctx: &mut ReasoningContext, + ) -> Result, Error>; + + /// Called when the LLM expresses tool intent without actually calling a tool. + /// Delegates can use this to emit events or log the nudge for observability. + async fn on_tool_intent_nudge(&self, _text: &str, _reason_ctx: &mut ReasoningContext) {} + + /// Called after each successful iteration (no error, no early return). + async fn after_iteration(&self, _iteration: usize) {} +} + +/// Run the unified agentic loop. +/// +/// This is the single implementation used by all three consumers (chat, job, container). +/// The `delegate` provides consumer-specific behavior via the `LoopDelegate` trait. +pub async fn run_agentic_loop( + delegate: &dyn LoopDelegate, + reasoning: &Reasoning, + reason_ctx: &mut ReasoningContext, + config: &AgenticLoopConfig, +) -> Result { + let mut consecutive_tool_intent_nudges: u32 = 0; + + for iteration in 1..=config.max_iterations { + // Check for external signals (stop, cancellation, user messages) + match delegate.check_signals().await { + LoopSignal::Continue => {} + LoopSignal::Stop => return Ok(LoopOutcome::Stopped), + LoopSignal::InjectMessage(msg) => { + reason_ctx.messages.push(ChatMessage::user(&msg)); + } + } + + // Pre-LLM call hook (cost guard, tool refresh, iteration limit nudge) + if let Some(outcome) = delegate.before_llm_call(reason_ctx, iteration).await { + return Ok(outcome); + } + + // Call LLM + let output = delegate.call_llm(reasoning, reason_ctx, iteration).await?; + + match output.result { + RespondResult::Text(text) => { + // Tool intent nudge: if the LLM says "let me search..." without + // actually calling a tool, inject a nudge message. + if config.enable_tool_intent_nudge + && !reason_ctx.available_tools.is_empty() + && !reason_ctx.force_text + && consecutive_tool_intent_nudges < config.max_tool_intent_nudges + && crate::llm::llm_signals_tool_intent(&text) + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + iteration, + "LLM expressed tool intent without calling a tool, nudging" + ); + delegate.on_tool_intent_nudge(&text, reason_ctx).await; + reason_ctx.messages.push(ChatMessage::assistant(&text)); + reason_ctx + .messages + .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + delegate.after_iteration(iteration).await; + continue; + } + + // Reset nudge counter since we got a non-intent text response + if !crate::llm::llm_signals_tool_intent(&text) { + consecutive_tool_intent_nudges = 0; + } + + match delegate.handle_text_response(&text, reason_ctx).await { + TextAction::Return(outcome) => return Ok(outcome), + TextAction::Continue => {} + } + } + RespondResult::ToolCalls { + tool_calls, + content, + } => { + consecutive_tool_intent_nudges = 0; + + if let Some(outcome) = delegate + .execute_tool_calls(tool_calls, content, reason_ctx) + .await? + { + return Ok(outcome); + } + } + } + + delegate.after_iteration(iteration).await; + } + + Ok(LoopOutcome::MaxIterations) +} + +/// Truncate a string for log/status previews. +/// +/// `max` is a byte budget. The result is truncated at the last valid char +/// boundary at or before `max` bytes, so it is always valid UTF-8. +pub fn truncate_for_preview(s: &str, max: usize) -> String { + if s.len() <= max { + s.to_string() + } else { + let end = crate::util::floor_char_boundary(s, max); + format!("{}...", &s[..end]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::llm::{RespondOutput, TokenUsage, ToolCall}; + use crate::testing::StubLlm; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::sync::Mutex; + + fn stub_reasoning() -> Reasoning { + Reasoning::new(Arc::new(StubLlm::default())) + } + + fn zero_usage() -> TokenUsage { + TokenUsage { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + } + } + + fn text_output(text: &str) -> RespondOutput { + RespondOutput { + result: RespondResult::Text(text.to_string()), + usage: zero_usage(), + } + } + + fn tool_calls_output(calls: Vec) -> RespondOutput { + RespondOutput { + result: RespondResult::ToolCalls { + tool_calls: calls, + content: None, + }, + usage: zero_usage(), + } + } + + /// Configurable mock delegate for testing run_agentic_loop. + struct MockDelegate { + signal: Mutex, + llm_responses: Mutex>, + tool_exec_count: AtomicUsize, + tool_exec_outcome: Mutex>, + iterations_seen: Mutex>, + early_exit: Mutex>, + nudge_count: AtomicUsize, + } + + impl MockDelegate { + fn new(responses: Vec) -> Self { + Self { + signal: Mutex::new(LoopSignal::Continue), + llm_responses: Mutex::new(responses), + tool_exec_count: AtomicUsize::new(0), + tool_exec_outcome: Mutex::new(None), + iterations_seen: Mutex::new(Vec::new()), + early_exit: Mutex::new(None), + nudge_count: AtomicUsize::new(0), + } + } + + fn with_signal(mut self, signal: LoopSignal) -> Self { + self.signal = Mutex::new(signal); + self + } + + fn with_early_exit(mut self, iteration: usize, outcome: LoopOutcome) -> Self { + self.early_exit = Mutex::new(Some((iteration, outcome))); + self + } + } + + #[async_trait] + impl LoopDelegate for MockDelegate { + async fn check_signals(&self) -> LoopSignal { + let mut sig = self.signal.lock().await; + std::mem::replace(&mut *sig, LoopSignal::Continue) + } + + async fn before_llm_call( + &self, + _reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Option { + let mut guard = self.early_exit.lock().await; + let should_take = guard + .as_ref() + .is_some_and(|(target, _)| *target == iteration); + if should_take { + guard.take().map(|(_, o)| o) + } else { + None + } + } + + async fn call_llm( + &self, + _reasoning: &Reasoning, + _reason_ctx: &mut ReasoningContext, + _iteration: usize, + ) -> Result { + let mut responses = self.llm_responses.lock().await; + if responses.is_empty() { + panic!("MockDelegate: no more LLM responses queued"); + } + Ok(responses.remove(0)) + } + + async fn handle_text_response( + &self, + text: &str, + _reason_ctx: &mut ReasoningContext, + ) -> TextAction { + TextAction::Return(LoopOutcome::Response(text.to_string())) + } + + async fn execute_tool_calls( + &self, + _tool_calls: Vec, + _content: Option, + reason_ctx: &mut ReasoningContext, + ) -> Result, crate::error::Error> { + self.tool_exec_count.fetch_add(1, Ordering::SeqCst); + reason_ctx + .messages + .push(ChatMessage::user("tool result stub")); + let outcome = self.tool_exec_outcome.lock().await.take(); + Ok(outcome) + } + + async fn on_tool_intent_nudge(&self, _text: &str, _reason_ctx: &mut ReasoningContext) { + self.nudge_count.fetch_add(1, Ordering::SeqCst); + } + + async fn after_iteration(&self, iteration: usize) { + self.iterations_seen.lock().await.push(iteration); + } + } + + // --- Tests --- + + #[tokio::test] + async fn test_text_response_returns_immediately() { + let delegate = MockDelegate::new(vec![text_output("Hello, world!")]); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + match outcome { + LoopOutcome::Response(text) => assert_eq!(text, "Hello, world!"), + _ => panic!("Expected LoopOutcome::Response"), + } + // after_iteration is NOT called when handle_text_response returns Return + // (the loop exits before reaching after_iteration). + assert!(delegate.iterations_seen.lock().await.is_empty()); + } + + #[tokio::test] + async fn test_tool_call_then_text_response() { + let tool_call = ToolCall { + id: "call_1".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({}), + }; + let delegate = MockDelegate::new(vec![ + tool_calls_output(vec![tool_call]), + text_output("Done!"), + ]); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + match outcome { + LoopOutcome::Response(text) => assert_eq!(text, "Done!"), + _ => panic!("Expected LoopOutcome::Response"), + } + assert_eq!(delegate.tool_exec_count.load(Ordering::SeqCst), 1); + // after_iteration called for iteration 1 (tool call), but not 2 + // (text response exits before after_iteration). + assert_eq!(*delegate.iterations_seen.lock().await, vec![1]); + } + + #[tokio::test] + async fn test_stop_signal_exits_immediately() { + let delegate = + MockDelegate::new(vec![text_output("unreachable")]).with_signal(LoopSignal::Stop); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + assert!(matches!(outcome, LoopOutcome::Stopped)); + assert!(delegate.iterations_seen.lock().await.is_empty()); + } + + #[tokio::test] + async fn test_inject_message_adds_user_message() { + let delegate = MockDelegate::new(vec![text_output("Got it")]) + .with_signal(LoopSignal::InjectMessage("injected prompt".to_string())); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + assert!(matches!(outcome, LoopOutcome::Response(_))); + assert!( + ctx.messages + .iter() + .any(|m| m.role == crate::llm::Role::User && m.content.contains("injected prompt")), + "Injected message should appear in context" + ); + } + + #[tokio::test] + async fn test_max_iterations_reached() { + struct ContinueDelegate; + + #[async_trait] + impl LoopDelegate for ContinueDelegate { + async fn check_signals(&self) -> LoopSignal { + LoopSignal::Continue + } + async fn before_llm_call( + &self, + _: &mut ReasoningContext, + _: usize, + ) -> Option { + None + } + async fn call_llm( + &self, + _: &Reasoning, + _: &mut ReasoningContext, + _: usize, + ) -> Result { + Ok(text_output("still working")) + } + async fn handle_text_response( + &self, + _: &str, + ctx: &mut ReasoningContext, + ) -> TextAction { + ctx.messages.push(ChatMessage::assistant("still working")); + TextAction::Continue + } + async fn execute_tool_calls( + &self, + _: Vec, + _: Option, + _: &mut ReasoningContext, + ) -> Result, crate::error::Error> { + Ok(None) + } + } + + let delegate = ContinueDelegate; + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig { + max_iterations: 3, + ..Default::default() + }; + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + assert!(matches!(outcome, LoopOutcome::MaxIterations)); + let assistant_count = ctx + .messages + .iter() + .filter(|m| m.role == crate::llm::Role::Assistant) + .count(); + assert_eq!(assistant_count, 3); + } + + #[tokio::test] + async fn test_tool_intent_nudge_fires_and_caps() { + let delegate = MockDelegate::new(vec![ + text_output("Let me search for that file"), + text_output("Let me search for that file"), + text_output("Let me search for that file"), + ]); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + ctx.available_tools.push(crate::llm::ToolDefinition { + name: "search".to_string(), + description: "Search files".to_string(), + parameters: serde_json::json!({"type": "object"}), + }); + let config = AgenticLoopConfig { + max_iterations: 10, + enable_tool_intent_nudge: true, + max_tool_intent_nudges: 2, + }; + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + assert!(matches!(outcome, LoopOutcome::Response(_))); + assert_eq!(delegate.nudge_count.load(Ordering::SeqCst), 2); + let nudge_messages = ctx + .messages + .iter() + .filter(|m| { + m.role == crate::llm::Role::User + && m.content.contains("you did not include any tool calls") + }) + .count(); + assert_eq!( + nudge_messages, 2, + "Should have exactly 2 nudge messages in context" + ); + } + + #[tokio::test] + async fn test_before_llm_call_early_exit() { + let delegate = MockDelegate::new(vec![text_output("unreachable")]) + .with_early_exit(1, LoopOutcome::Stopped); + let reasoning = stub_reasoning(); + let mut ctx = ReasoningContext::new(); + let config = AgenticLoopConfig::default(); + + let outcome = run_agentic_loop(&delegate, &reasoning, &mut ctx, &config) + .await + .unwrap(); + + assert!(matches!(outcome, LoopOutcome::Stopped)); + assert!(delegate.iterations_seen.lock().await.is_empty()); + } + + #[test] + fn test_truncate_short_string_unchanged() { + assert_eq!(truncate_for_preview("hello", 10), "hello"); + } + + #[test] + fn test_truncate_long_string_adds_ellipsis() { + let result = truncate_for_preview("hello world", 5); + assert_eq!(result, "hello..."); + } + + #[test] + fn test_truncate_multibyte_safe() { + let result = truncate_for_preview("café", 4); + assert_eq!(result, "caf..."); + } +} diff --git a/src/agent/commands.rs b/src/agent/commands.rs index 2c5b96e5..90266d0b 100644 --- a/src/agent/commands.rs +++ b/src/agent/commands.rs @@ -405,7 +405,8 @@ impl Agent { .with_max_tokens(512) .with_temperature(0.3); - let reasoning = Reasoning::new(self.llm().clone()); + let reasoning = + Reasoning::new(self.llm().clone()).with_model_name(self.llm().active_model_name()); match reasoning.complete(request).await { Ok((text, _usage)) => Ok(SubmissionResult::response(format!( "Thread Summary:\n\n{}", @@ -453,7 +454,8 @@ impl Agent { .with_max_tokens(512) .with_temperature(0.5); - let reasoning = Reasoning::new(self.llm().clone()); + let reasoning = + Reasoning::new(self.llm().clone()).with_model_name(self.llm().active_model_name()); match reasoning.complete(request).await { Ok((text, _usage)) => Ok(SubmissionResult::response(format!( "Suggested Next Steps:\n\n{}", diff --git a/src/agent/compaction.rs b/src/agent/compaction.rs index 583d92de..30bb2b6c 100644 --- a/src/agent/compaction.rs +++ b/src/agent/compaction.rs @@ -227,7 +227,8 @@ Be brief but capture all important details. Use bullet points."#, .with_max_tokens(1024) .with_temperature(0.3); - let reasoning = Reasoning::new(self.llm.clone()); + let reasoning = + Reasoning::new(self.llm.clone()).with_model_name(self.llm.active_model_name()); let (text, _) = reasoning.complete(request).await?; Ok(text) } diff --git a/src/agent/dispatcher.rs b/src/agent/dispatcher.rs index 99feed9d..b791f6d7 100644 --- a/src/agent/dispatcher.rs +++ b/src/agent/dispatcher.rs @@ -14,7 +14,12 @@ use crate::agent::session::{PendingApproval, Session, ThreadState}; use crate::channels::{IncomingMessage, StatusUpdate}; use crate::context::JobContext; use crate::error::Error; -use crate::llm::{ChatMessage, Reasoning, ReasoningContext, RespondResult}; +use async_trait::async_trait; + +use crate::agent::agentic_loop::{ + AgenticLoopConfig, LoopDelegate, LoopOutcome, LoopSignal, TextAction, +}; +use crate::llm::{ChatMessage, Reasoning, ReasoningContext}; use crate::tools::redact_params; /// Result of the agentic loop execution. @@ -85,7 +90,7 @@ impl Agent { crate::skills::SkillTrust::Installed => "INSTALLED", }; - tracing::info!( + tracing::debug!( skill_name = skill.name(), skill_version = skill.version(), trust = %skill.trust, @@ -133,9 +138,6 @@ impl Agent { reasoning = reasoning.with_skill_context(ctx); } - // Build context with messages that we'll mutate during the loop - let mut context_messages = initial_messages; - // Create a JobContext for tool execution (chat doesn't have a real job) let mut job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session"); @@ -154,700 +156,62 @@ impl Agent { let cached_prompt_no_tools = reasoning.build_system_prompt_with_tools(&[]); let max_tool_iterations = self.config.max_tool_iterations; - // Force a text-only response on the last iteration to guarantee termination - // instead of hard-erroring. The penultimate iteration also gets a nudge - // message so the LLM knows it should wrap up. let force_text_at = max_tool_iterations; let nudge_at = max_tool_iterations.saturating_sub(1); - let mut iteration = 0; - const MAX_TOOL_INTENT_NUDGES: u32 = 2; - let mut consecutive_tool_intent_nudges: u32 = 0; - loop { - iteration += 1; - // Hard ceiling one past the forced-text iteration (should never be reached - // since force_text_at guarantees a text response, but kept as a safety net). - if iteration > max_tool_iterations + 1 { - return Err(crate::error::LlmError::InvalidResponse { - provider: "agent".to_string(), - reason: format!("Exceeded maximum tool iterations ({max_tool_iterations})"), - } - .into()); + + let delegate = ChatDelegate { + agent: self, + session: session.clone(), + thread_id, + message, + job_ctx, + active_skills, + cached_prompt, + cached_prompt_no_tools, + nudge_at, + force_text_at, + user_tz, + }; + + let mut reason_ctx = ReasoningContext::new() + .with_messages(initial_messages) + .with_tools(initial_tool_defs) + .with_system_prompt(delegate.cached_prompt.clone()) + .with_metadata({ + let mut m = std::collections::HashMap::new(); + m.insert("thread_id".to_string(), thread_id.to_string()); + m + }); + + let loop_config = AgenticLoopConfig { + // Hard ceiling: one past force_text_at (safety net). + max_iterations: max_tool_iterations + 1, + enable_tool_intent_nudge: true, + max_tool_intent_nudges: 2, + }; + + let outcome = crate::agent::agentic_loop::run_agentic_loop( + &delegate, + &reasoning, + &mut reason_ctx, + &loop_config, + ) + .await?; + + match outcome { + LoopOutcome::Response(text) => Ok(AgenticLoopResult::Response(text)), + LoopOutcome::Stopped => Err(crate::error::JobError::ContextError { + id: thread_id, + reason: "Interrupted".to_string(), } - - // Check if interrupted - { - let sess = session.lock().await; - if let Some(thread) = sess.threads.get(&thread_id) - && thread.state == ThreadState::Interrupted - { - return Err(crate::error::JobError::ContextError { - id: thread_id, - reason: "Interrupted".to_string(), - } - .into()); - } + .into()), + LoopOutcome::MaxIterations => Err(crate::error::LlmError::InvalidResponse { + provider: "agent".to_string(), + reason: format!("Exceeded maximum tool iterations ({max_tool_iterations})"), } - - // Enforce cost guardrails before the LLM call - if let Err(limit) = self.cost_guard().check_allowed().await { - return Err(crate::error::LlmError::InvalidResponse { - provider: "agent".to_string(), - reason: limit.to_string(), - } - .into()); - } - - // Inject a nudge message when approaching the iteration limit so the - // LLM is aware it should produce a final answer on the next turn. - if iteration == nudge_at { - context_messages.push(ChatMessage::system( - "You are approaching the tool call limit. \ - Provide your best final answer on the next response \ - using the information you have gathered so far. \ - Do not call any more tools.", - )); - } - - let force_text = iteration >= force_text_at; - - // Refresh tool definitions each iteration so newly built tools become visible - let tool_defs = self.tools().tool_definitions().await; - - // Apply trust-based tool attenuation if skills are active. - let tool_defs = if !active_skills.is_empty() { - let result = crate::skills::attenuate_tools(&tool_defs, &active_skills); - tracing::info!( - min_trust = %result.min_trust, - tools_available = result.tools.len(), - tools_removed = result.removed_tools.len(), - removed = ?result.removed_tools, - explanation = %result.explanation, - "Tool attenuation applied" - ); - result.tools - } else { - tool_defs - }; - - // Call LLM with current context; force_text drops tools to guarantee a - // text response on the final iteration. The pre-built system prompt - // avoids rebuilding the same ~1,500-token string each iteration. - let mut context = ReasoningContext::new() - .with_messages(context_messages.clone()) - .with_tools(tool_defs) - .with_system_prompt(if force_text { - cached_prompt_no_tools.clone() - } else { - cached_prompt.clone() - }) - .with_metadata({ - let mut m = std::collections::HashMap::new(); - m.insert("thread_id".to_string(), thread_id.to_string()); - m - }); - context.force_text = force_text; - - if force_text { - tracing::info!( - iteration, - "Forcing text-only response (iteration limit reached)" - ); - } - - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::Thinking("Calling LLM...".into()), - &message.metadata, - ) - .await; - - let output = match reasoning.respond_with_tools(&context).await { - Ok(output) => output, - Err(crate::error::LlmError::ContextLengthExceeded { used, limit }) => { - tracing::warn!( - used, - limit, - iteration, - "Context length exceeded, compacting messages and retrying" - ); - - // Compact: keep system messages + last user message + current turn - context_messages = compact_messages_for_retry(&context_messages); - - // Rebuild context with compacted messages, reusing cached prompt - let mut retry_context = ReasoningContext::new() - .with_messages(context_messages.clone()) - .with_tools(if force_text { - Vec::new() - } else { - context.available_tools.clone() - }) - .with_metadata(context.metadata.clone()); - retry_context.force_text = force_text; - retry_context.system_prompt = context.system_prompt.clone(); - - reasoning - .respond_with_tools(&retry_context) - .await - .map_err(|retry_err| { - tracing::error!( - original_used = used, - original_limit = limit, - retry_error = %retry_err, - "Retry after auto-compaction also failed" - ); - // Propagate the actual retry error so callers see the real failure - crate::error::Error::from(retry_err) - })? - } - Err(e) => return Err(e.into()), - }; - - // Record cost and track token usage - let model_name = self.llm().active_model_name(); - let read_discount = self.llm().cache_read_discount(); - let write_multiplier = self.llm().cache_write_multiplier(); - let call_cost = self - .cost_guard() - .record_llm_call( - &model_name, - output.usage.input_tokens, - output.usage.output_tokens, - output.usage.cache_read_input_tokens, - output.usage.cache_creation_input_tokens, - read_discount, - write_multiplier, - Some(self.llm().cost_per_token()), - ) - .await; - tracing::debug!( - "LLM call used {} input + {} output tokens (${:.6})", - output.usage.input_tokens, - output.usage.output_tokens, - call_cost, - ); - - match output.result { - RespondResult::Text(text) => { - // Nudge the LLM if it expressed tool intent without calling tools. - // This is common with non-Anthropic models (e.g. GLM-5 via NEAR AI) - // that output "Let me search…" but don't issue tool_calls. - if !force_text - && !context.available_tools.is_empty() - && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES - && crate::llm::llm_signals_tool_intent(&text) - { - consecutive_tool_intent_nudges += 1; - tracing::info!( - iteration, - "LLM expressed tool intent without calling a tool, nudging" - ); - context_messages.push(ChatMessage::assistant(&text)); - context_messages.push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); - continue; - } - - // Strip internal "[Called tool ...]" text that can leak when - // provider flattening (e.g. NEAR AI) converts tool_calls to - // plain text and the LLM echoes it back. - let sanitized = strip_internal_tool_call_text(&text); - return Ok(AgenticLoopResult::Response(sanitized)); - } - RespondResult::ToolCalls { - tool_calls, - content, - } => { - consecutive_tool_intent_nudges = 0; - // Add the assistant message with tool_calls to context. - // OpenAI protocol requires this before tool-result messages. - context_messages.push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); - - // Execute tools and add results to context - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::Thinking(format!( - "Executing {} tool(s)...", - tool_calls.len() - )), - &message.metadata, - ) - .await; - - // Record tool calls in the thread with sensitive params redacted. - // Look up each tool's sensitive_params before acquiring the session lock. - { - let mut redacted_args: Vec = - Vec::with_capacity(tool_calls.len()); - for tc in &tool_calls { - let safe = if let Some(tool) = self.tools().get(&tc.name).await { - redact_params(&tc.arguments, tool.sensitive_params()) - } else { - tc.arguments.clone() - }; - redacted_args.push(safe); - } - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) - && let Some(turn) = thread.last_turn_mut() - { - for (tc, safe_args) in tool_calls.iter().zip(redacted_args) { - turn.record_tool_call(&tc.name, safe_args); - } - } - } - - // === Phase 1: Preflight (sequential) === - // Walk tool_calls checking approval and hooks. Classify - // each tool as Rejected (by hook) or Runnable. Stop at the - // first tool that needs approval. - // - // Outcomes are indexed by original tool_calls position so - // Phase 3 can emit results in the correct order. - enum PreflightOutcome { - /// Hook rejected/blocked this tool; contains the error message. - Rejected(String), - /// Tool passed preflight and will be executed. - Runnable, - } - let mut preflight: Vec<(crate::llm::ToolCall, PreflightOutcome)> = Vec::new(); - let mut runnable: Vec<(usize, crate::llm::ToolCall)> = Vec::new(); - let mut approval_needed: Option<( - usize, - crate::llm::ToolCall, - Arc, - )> = None; - - for (idx, original_tc) in tool_calls.iter().enumerate() { - let mut tc = original_tc.clone(); - - // Fetch the tool upfront so we can redact sensitive params - // before they touch hooks or approval display. - let tool_opt = self.tools().get(&tc.name).await; - let sensitive = tool_opt - .as_ref() - .map(|t| t.sensitive_params()) - .unwrap_or(&[]); - - // Hook: BeforeToolCall (runs before approval so hooks can - // modify parameters — approval is checked on final params). - // Hooks receive redacted params so sensitive values are not - // exposed to hook handlers or their logs. - let hook_params = redact_params(&tc.arguments, sensitive); - let event = crate::hooks::HookEvent::ToolCall { - tool_name: tc.name.clone(), - parameters: hook_params, - user_id: message.user_id.clone(), - context: "chat".to_string(), - }; - match self.hooks().run(&event).await { - Err(crate::hooks::HookError::Rejected { reason }) => { - preflight.push(( - tc, - PreflightOutcome::Rejected(format!( - "Tool call rejected by hook: {}", - reason - )), - )); - continue; // skip to next tool (not infinite: using for loop) - } - Err(err) => { - preflight.push(( - tc, - PreflightOutcome::Rejected(format!( - "Tool call blocked by hook policy: {}", - err - )), - )); - continue; - } - Ok(crate::hooks::HookOutcome::Continue { - modified: Some(new_params), - }) => match serde_json::from_str::(&new_params) { - Ok(mut parsed) => { - // Restore original sensitive param values so a hook - // cannot overwrite them (they were sent as [REDACTED]). - if let Some(obj) = parsed.as_object_mut() { - for key in sensitive { - if let Some(orig_val) = original_tc.arguments.get(*key) - { - obj.insert((*key).to_string(), orig_val.clone()); - } - } - } - tc.arguments = parsed; - } - Err(e) => { - tracing::warn!( - tool = %tc.name, - "Hook returned non-JSON modification for ToolCall, ignoring: {}", - e - ); - } - }, - _ => {} - } - - // Check if tool requires approval on the final (post-hook) - // parameters. Skipped when auto_approve_tools is set. - if !self.config.auto_approve_tools - && let Some(tool) = tool_opt - { - use crate::tools::ApprovalRequirement; - let needs_approval = match tool.requires_approval(&tc.arguments) { - ApprovalRequirement::Never => false, - ApprovalRequirement::UnlessAutoApproved => { - let sess = session.lock().await; - !sess.is_tool_auto_approved(&tc.name) - } - ApprovalRequirement::Always => true, - }; - - if needs_approval { - approval_needed = Some((idx, tc, tool)); - break; // remaining tools are deferred - } - } - - let preflight_idx = preflight.len(); - preflight.push((tc.clone(), PreflightOutcome::Runnable)); - runnable.push((preflight_idx, tc)); - } - - // === Phase 2: Parallel execution === - // Execute runnable tools and slot results back by preflight - // index so Phase 3 can iterate in original order. - let mut exec_results: Vec>> = - (0..preflight.len()).map(|_| None).collect(); - - if runnable.len() <= 1 { - // Single tool (or none): execute inline - for (pf_idx, tc) in &runnable { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolStarted { - name: tc.name.clone(), - }, - &message.metadata, - ) - .await; - - let result = self - .execute_chat_tool(&tc.name, &tc.arguments, &job_ctx) - .await; - - let disp_tool = self.tools().get(&tc.name).await; - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::tool_completed( - tc.name.clone(), - &result, - &tc.arguments, - disp_tool.as_deref(), - ), - &message.metadata, - ) - .await; - - exec_results[*pf_idx] = Some(result); - } - } else { - // Multiple tools: execute in parallel via JoinSet - let mut join_set = JoinSet::new(); - - for (pf_idx, tc) in &runnable { - let pf_idx = *pf_idx; - let tools = self.tools().clone(); - let safety = self.safety().clone(); - let channels = self.channels.clone(); - let job_ctx = job_ctx.clone(); - let tc = tc.clone(); - let channel = message.channel.clone(); - let metadata = message.metadata.clone(); - - join_set.spawn(async move { - let _ = channels - .send_status( - &channel, - StatusUpdate::ToolStarted { - name: tc.name.clone(), - }, - &metadata, - ) - .await; - - let result = execute_chat_tool_standalone( - &tools, - &safety, - &tc.name, - &tc.arguments, - &job_ctx, - ) - .await; - - let par_tool = tools.get(&tc.name).await; - let _ = channels - .send_status( - &channel, - StatusUpdate::tool_completed( - tc.name.clone(), - &result, - &tc.arguments, - par_tool.as_deref(), - ), - &metadata, - ) - .await; - - (pf_idx, result) - }); - } - - while let Some(join_result) = join_set.join_next().await { - match join_result { - Ok((pf_idx, result)) => { - exec_results[pf_idx] = Some(result); - } - Err(e) => { - if e.is_panic() { - tracing::error!("Chat tool execution task panicked: {}", e); - } else { - tracing::error!( - "Chat tool execution task cancelled: {}", - e - ); - } - } - } - } - - // Fill panicked slots with error results - for (runnable_idx, (pf_idx, tc)) in runnable.iter().enumerate() { - if exec_results[*pf_idx].is_none() { - tracing::error!( - tool = %tc.name, - runnable_idx, - "Filling failed task slot with error" - ); - exec_results[*pf_idx] = - Some(Err(crate::error::ToolError::ExecutionFailed { - name: tc.name.clone(), - reason: "Task failed during execution".to_string(), - } - .into())); - } - } - } - - // === Phase 3: Post-flight (sequential, in original order) === - // Process all results — both hook rejections and execution - // results — in the original tool_calls order. Auth intercept - // is deferred until after every result is recorded. - let mut deferred_auth: Option = None; - - for (pf_idx, (tc, outcome)) in preflight.into_iter().enumerate() { - match outcome { - PreflightOutcome::Rejected(error_msg) => { - // Record hook rejection in thread - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) - && let Some(turn) = thread.last_turn_mut() - { - turn.record_tool_error(error_msg.clone()); - } - } - context_messages - .push(ChatMessage::tool_result(&tc.id, &tc.name, error_msg)); - } - PreflightOutcome::Runnable => { - // Retrieve the execution result for this slot - let tool_result = - exec_results[pf_idx].take().unwrap_or_else(|| { - Err(crate::error::ToolError::ExecutionFailed { - name: tc.name.clone(), - reason: "No result available".to_string(), - } - .into()) - }); - - // Detect image generation sentinel in tool output - // (only from image tools — avoids parsing all tool outputs) - let is_image_sentinel = if let Ok(ref output) = tool_result - && matches!(tc.name.as_str(), "image_generate" | "image_edit") - { - if let Ok(sentinel) = - serde_json::from_str::(output) - && sentinel.get("type").and_then(|v| v.as_str()) - == Some("image_generated") - { - let data_url = sentinel - .get("data") - .and_then(|v| v.as_str()) - .unwrap_or_default() - .to_string(); - let path = sentinel - .get("path") - .and_then(|v| v.as_str()) - .map(String::from); - // Skip broadcasting if data_url is empty to avoid - // sending a broken ImageGenerated SSE event. - if data_url.is_empty() { - tracing::warn!( - "Image generation sentinel has empty data URL, skipping broadcast" - ); - } else { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ImageGenerated { data_url, path }, - &message.metadata, - ) - .await; - } - true - } else { - false - } - } else { - false - }; - - // Send ToolResult preview (skip for image sentinels to avoid - // broadcasting multi-MB base64 data as a preview) - if !is_image_sentinel - && let Ok(ref output) = tool_result - && !output.is_empty() - { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolResult { - name: tc.name.clone(), - preview: output.clone(), - }, - &message.metadata, - ) - .await; - } - - // Check for auth awaiting — defer the return - // until all results are recorded. - if deferred_auth.is_none() - && let Some((ext_name, instructions)) = - check_auth_required(&tc.name, &tool_result) - { - let auth_data = parse_auth_result(&tool_result); - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.enter_auth_mode(ext_name.clone()); - } - } - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::AuthRequired { - extension_name: ext_name, - instructions: Some(instructions.clone()), - auth_url: auth_data.auth_url, - setup_url: auth_data.setup_url, - }, - &message.metadata, - ) - .await; - deferred_auth = Some(instructions); - } - - // Stash full output so subsequent tools can reference it - if let Ok(ref output) = tool_result { - job_ctx - .tool_output_stash - .write() - .await - .insert(tc.id.clone(), output.clone()); - } - - // Sanitize and add tool result to context - let is_tool_error = tool_result.is_err(); - let result_content = match tool_result { - Ok(output) => { - let sanitized = - self.safety().sanitize_tool_output(&tc.name, &output); - self.safety().wrap_for_llm( - &tc.name, - &sanitized.content, - sanitized.was_modified, - ) - } - Err(e) => format!("Tool '{}' failed: {}", tc.name, e), - }; - - // Record sanitized result in thread so messages() - // and persist_tool_calls() use cleaned content. - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) - && let Some(turn) = thread.last_turn_mut() - { - if is_tool_error { - turn.record_tool_error(result_content.clone()); - } else { - turn.record_tool_result(serde_json::json!( - result_content - )); - } - } - } - - context_messages.push(ChatMessage::tool_result( - &tc.id, - &tc.name, - result_content, - )); - } - } - } - - // Return auth response after all results are recorded - if let Some(instructions) = deferred_auth { - return Ok(AgenticLoopResult::Response(instructions)); - } - - // Handle approval if a tool needed it - if let Some((approval_idx, tc, tool)) = approval_needed { - // Show redacted params in the approval UI — the user already knows - // the sensitive value (they provided it); showing it again is - // unnecessary and creates a leakage path through channel logs. - let display_params = redact_params(&tc.arguments, tool.sensitive_params()); - let pending = PendingApproval { - request_id: Uuid::new_v4(), - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - display_parameters: display_params, - description: tool.description().to_string(), - tool_call_id: tc.id.clone(), - context_messages: context_messages.clone(), - deferred_tool_calls: tool_calls[approval_idx + 1..].to_vec(), - user_timezone: Some(user_tz.name().to_string()), - }; - - return Ok(AgenticLoopResult::NeedApproval { pending }); - } - } + .into()), + LoopOutcome::NeedApproval(pending) => { + Ok(AgenticLoopResult::NeedApproval { pending: *pending }) } } } @@ -863,11 +227,660 @@ impl Agent { } } +/// Delegate for the chat (dispatcher) context. +/// +/// Implements `LoopDelegate` to customize the shared agentic loop for +/// interactive chat sessions with the full 3-phase tool execution +/// (preflight → parallel exec → post-flight), approval flow, hooks, +/// auth intercept, and cost tracking. +struct ChatDelegate<'a> { + agent: &'a Agent, + session: Arc>, + thread_id: Uuid, + message: &'a IncomingMessage, + job_ctx: JobContext, + active_skills: Vec, + cached_prompt: String, + cached_prompt_no_tools: String, + nudge_at: usize, + force_text_at: usize, + user_tz: chrono_tz::Tz, +} + +#[async_trait] +impl<'a> LoopDelegate for ChatDelegate<'a> { + async fn check_signals(&self) -> LoopSignal { + let sess = self.session.lock().await; + if let Some(thread) = sess.threads.get(&self.thread_id) + && thread.state == ThreadState::Interrupted + { + return LoopSignal::Stop; + } + LoopSignal::Continue + } + + async fn before_llm_call( + &self, + reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Option { + // Inject a nudge message when approaching the iteration limit so the + // LLM is aware it should produce a final answer on the next turn. + if iteration == self.nudge_at { + reason_ctx.messages.push(ChatMessage::system( + "You are approaching the tool call limit. \ + Provide your best final answer on the next response \ + using the information you have gathered so far. \ + Do not call any more tools.", + )); + } + + let force_text = iteration >= self.force_text_at; + + // Refresh tool definitions each iteration so newly built tools become visible + let tool_defs = self.agent.tools().tool_definitions().await; + + // Apply trust-based tool attenuation if skills are active. + let tool_defs = if !self.active_skills.is_empty() { + let result = crate::skills::attenuate_tools(&tool_defs, &self.active_skills); + tracing::debug!( + min_trust = %result.min_trust, + tools_available = result.tools.len(), + tools_removed = result.removed_tools.len(), + removed = ?result.removed_tools, + explanation = %result.explanation, + "Tool attenuation applied" + ); + result.tools + } else { + tool_defs + }; + + // Update context for this iteration + reason_ctx.available_tools = tool_defs; + reason_ctx.system_prompt = Some(if force_text { + self.cached_prompt_no_tools.clone() + } else { + self.cached_prompt.clone() + }); + reason_ctx.force_text = force_text; + + if force_text { + tracing::info!( + iteration, + "Forcing text-only response (iteration limit reached)" + ); + } + + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::Thinking("Calling LLM...".into()), + &self.message.metadata, + ) + .await; + + None + } + + async fn call_llm( + &self, + reasoning: &Reasoning, + reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Result { + // Enforce cost guardrails before the LLM call + if let Err(limit) = self.agent.cost_guard().check_allowed().await { + return Err(crate::error::LlmError::InvalidResponse { + provider: "agent".to_string(), + reason: limit.to_string(), + } + .into()); + } + + let output = match reasoning.respond_with_tools(reason_ctx).await { + Ok(output) => output, + Err(crate::error::LlmError::ContextLengthExceeded { used, limit }) => { + tracing::warn!( + used, + limit, + iteration, + "Context length exceeded, compacting messages and retrying" + ); + + // Compact messages in place and retry + reason_ctx.messages = compact_messages_for_retry(&reason_ctx.messages); + + // When force_text, clear tools to further reduce token count + if reason_ctx.force_text { + reason_ctx.available_tools.clear(); + } + + reasoning + .respond_with_tools(reason_ctx) + .await + .map_err(|retry_err| { + tracing::error!( + original_used = used, + original_limit = limit, + retry_error = %retry_err, + "Retry after auto-compaction also failed" + ); + crate::error::Error::from(retry_err) + })? + } + Err(e) => return Err(e.into()), + }; + + // Record cost and track token usage + let model_name = self.agent.llm().active_model_name(); + let read_discount = self.agent.llm().cache_read_discount(); + let write_multiplier = self.agent.llm().cache_write_multiplier(); + let call_cost = self + .agent + .cost_guard() + .record_llm_call( + &model_name, + output.usage.input_tokens, + output.usage.output_tokens, + output.usage.cache_read_input_tokens, + output.usage.cache_creation_input_tokens, + read_discount, + write_multiplier, + Some(self.agent.llm().cost_per_token()), + ) + .await; + tracing::debug!( + "LLM call used {} input + {} output tokens (${:.6})", + output.usage.input_tokens, + output.usage.output_tokens, + call_cost, + ); + + Ok(output) + } + + async fn handle_text_response( + &self, + text: &str, + _reason_ctx: &mut ReasoningContext, + ) -> TextAction { + // Strip internal "[Called tool ...]" text that can leak when + // provider flattening (e.g. NEAR AI) converts tool_calls to + // plain text and the LLM echoes it back. + let sanitized = strip_internal_tool_call_text(text); + TextAction::Return(LoopOutcome::Response(sanitized)) + } + + async fn execute_tool_calls( + &self, + tool_calls: Vec, + content: Option, + reason_ctx: &mut ReasoningContext, + ) -> Result, Error> { + // Add the assistant message with tool_calls to context. + // OpenAI protocol requires this before tool-result messages. + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls( + content, + tool_calls.clone(), + )); + + // Execute tools and add results to context + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::Thinking(format!("Executing {} tool(s)...", tool_calls.len())), + &self.message.metadata, + ) + .await; + + // Record tool calls in the thread with sensitive params redacted. + { + let mut redacted_args: Vec = Vec::with_capacity(tool_calls.len()); + for tc in &tool_calls { + let safe = if let Some(tool) = self.agent.tools().get(&tc.name).await { + redact_params(&tc.arguments, tool.sensitive_params()) + } else { + tc.arguments.clone() + }; + redacted_args.push(safe); + } + let mut sess = self.session.lock().await; + if let Some(thread) = sess.threads.get_mut(&self.thread_id) + && let Some(turn) = thread.last_turn_mut() + { + for (tc, safe_args) in tool_calls.iter().zip(redacted_args) { + turn.record_tool_call(&tc.name, safe_args); + } + } + } + + // === Phase 1: Preflight (sequential) === + // Walk tool_calls checking approval and hooks. Classify + // each tool as Rejected (by hook) or Runnable. Stop at the + // first tool that needs approval. + enum PreflightOutcome { + Rejected(String), + Runnable, + } + let mut preflight: Vec<(crate::llm::ToolCall, PreflightOutcome)> = Vec::new(); + let mut runnable: Vec<(usize, crate::llm::ToolCall)> = Vec::new(); + let mut approval_needed: Option<( + usize, + crate::llm::ToolCall, + Arc, + )> = None; + + for (idx, original_tc) in tool_calls.iter().enumerate() { + let mut tc = original_tc.clone(); + + let tool_opt = self.agent.tools().get(&tc.name).await; + let sensitive = tool_opt + .as_ref() + .map(|t| t.sensitive_params()) + .unwrap_or(&[]); + + // Hook: BeforeToolCall + let hook_params = redact_params(&tc.arguments, sensitive); + let event = crate::hooks::HookEvent::ToolCall { + tool_name: tc.name.clone(), + parameters: hook_params, + user_id: self.message.user_id.clone(), + context: "chat".to_string(), + }; + match self.agent.hooks().run(&event).await { + Err(crate::hooks::HookError::Rejected { reason }) => { + preflight.push(( + tc, + PreflightOutcome::Rejected(format!( + "Tool call rejected by hook: {}", + reason + )), + )); + continue; + } + Err(err) => { + preflight.push(( + tc, + PreflightOutcome::Rejected(format!( + "Tool call blocked by hook policy: {}", + err + )), + )); + continue; + } + Ok(crate::hooks::HookOutcome::Continue { + modified: Some(new_params), + }) => match serde_json::from_str::(&new_params) { + Ok(mut parsed) => { + if let Some(obj) = parsed.as_object_mut() { + for key in sensitive { + if let Some(orig_val) = original_tc.arguments.get(*key) { + obj.insert((*key).to_string(), orig_val.clone()); + } + } + } + tc.arguments = parsed; + } + Err(e) => { + tracing::warn!( + tool = %tc.name, + "Hook returned non-JSON modification for ToolCall, ignoring: {}", + e + ); + } + }, + _ => {} + } + + // Check if tool requires approval + if !self.agent.config.auto_approve_tools + && let Some(tool) = tool_opt + { + use crate::tools::ApprovalRequirement; + let needs_approval = match tool.requires_approval(&tc.arguments) { + ApprovalRequirement::Never => false, + ApprovalRequirement::UnlessAutoApproved => { + let sess = self.session.lock().await; + !sess.is_tool_auto_approved(&tc.name) + } + ApprovalRequirement::Always => true, + }; + + if needs_approval { + approval_needed = Some((idx, tc, tool)); + break; + } + } + + let preflight_idx = preflight.len(); + preflight.push((tc.clone(), PreflightOutcome::Runnable)); + runnable.push((preflight_idx, tc)); + } + + // === Phase 2: Parallel execution === + let mut exec_results: Vec>> = + (0..preflight.len()).map(|_| None).collect(); + + if runnable.len() <= 1 { + for (pf_idx, tc) in &runnable { + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &self.message.metadata, + ) + .await; + + let result = self + .agent + .execute_chat_tool(&tc.name, &tc.arguments, &self.job_ctx) + .await; + + let disp_tool = self.agent.tools().get(&tc.name).await; + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + disp_tool.as_deref(), + ), + &self.message.metadata, + ) + .await; + + exec_results[*pf_idx] = Some(result); + } + } else { + let mut join_set = JoinSet::new(); + + for (pf_idx, tc) in &runnable { + let pf_idx = *pf_idx; + let tools = self.agent.tools().clone(); + let safety = self.agent.safety().clone(); + let channels = self.agent.channels.clone(); + let job_ctx = self.job_ctx.clone(); + let tc = tc.clone(); + let channel = self.message.channel.clone(); + let metadata = self.message.metadata.clone(); + + join_set.spawn(async move { + let _ = channels + .send_status( + &channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &metadata, + ) + .await; + + let result = execute_chat_tool_standalone( + &tools, + &safety, + &tc.name, + &tc.arguments, + &job_ctx, + ) + .await; + + let par_tool = tools.get(&tc.name).await; + let _ = channels + .send_status( + &channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + par_tool.as_deref(), + ), + &metadata, + ) + .await; + + (pf_idx, result) + }); + } + + while let Some(join_result) = join_set.join_next().await { + match join_result { + Ok((pf_idx, result)) => { + exec_results[pf_idx] = Some(result); + } + Err(e) => { + if e.is_panic() { + tracing::error!("Chat tool execution task panicked: {}", e); + } else { + tracing::error!("Chat tool execution task cancelled: {}", e); + } + } + } + } + + // Fill panicked slots with error results + for (pf_idx, tc) in runnable.iter() { + if exec_results[*pf_idx].is_none() { + tracing::error!( + tool = %tc.name, + "Filling failed task slot with error" + ); + exec_results[*pf_idx] = Some(Err(crate::error::ToolError::ExecutionFailed { + name: tc.name.clone(), + reason: "Task failed during execution".to_string(), + } + .into())); + } + } + } + + // === Phase 3: Post-flight (sequential, in original order) === + let mut deferred_auth: Option = None; + + for (pf_idx, (tc, outcome)) in preflight.into_iter().enumerate() { + match outcome { + PreflightOutcome::Rejected(error_msg) => { + { + let mut sess = self.session.lock().await; + if let Some(thread) = sess.threads.get_mut(&self.thread_id) + && let Some(turn) = thread.last_turn_mut() + { + turn.record_tool_error(error_msg.clone()); + } + } + reason_ctx + .messages + .push(ChatMessage::tool_result(&tc.id, &tc.name, error_msg)); + } + PreflightOutcome::Runnable => { + let tool_result = exec_results[pf_idx].take().unwrap_or_else(|| { + Err(crate::error::ToolError::ExecutionFailed { + name: tc.name.clone(), + reason: "No result available".to_string(), + } + .into()) + }); + + // Detect image generation sentinel + let is_image_sentinel = if let Ok(ref output) = tool_result + && matches!(tc.name.as_str(), "image_generate" | "image_edit") + { + if let Ok(sentinel) = serde_json::from_str::(output) + && sentinel.get("type").and_then(|v| v.as_str()) + == Some("image_generated") + { + let data_url = sentinel + .get("data") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string(); + let path = sentinel + .get("path") + .and_then(|v| v.as_str()) + .map(String::from); + if data_url.is_empty() { + tracing::warn!( + "Image generation sentinel has empty data URL, skipping broadcast" + ); + } else { + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::ImageGenerated { data_url, path }, + &self.message.metadata, + ) + .await; + } + true + } else { + false + } + } else { + false + }; + + // Send ToolResult preview + if !is_image_sentinel + && let Ok(ref output) = tool_result + && !output.is_empty() + { + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::ToolResult { + name: tc.name.clone(), + preview: output.clone(), + }, + &self.message.metadata, + ) + .await; + } + + // Check for auth awaiting + if deferred_auth.is_none() + && let Some((ext_name, instructions)) = + check_auth_required(&tc.name, &tool_result) + { + let auth_data = parse_auth_result(&tool_result); + { + let mut sess = self.session.lock().await; + if let Some(thread) = sess.threads.get_mut(&self.thread_id) { + thread.enter_auth_mode(ext_name.clone()); + } + } + let _ = self + .agent + .channels + .send_status( + &self.message.channel, + StatusUpdate::AuthRequired { + extension_name: ext_name, + instructions: Some(instructions.clone()), + auth_url: auth_data.auth_url, + setup_url: auth_data.setup_url, + }, + &self.message.metadata, + ) + .await; + deferred_auth = Some(instructions); + } + + // Stash full output so subsequent tools can reference it + if let Ok(ref output) = tool_result { + self.job_ctx + .tool_output_stash + .write() + .await + .insert(tc.id.clone(), output.clone()); + } + + // Sanitize and add tool result to context + let is_tool_error = tool_result.is_err(); + let result_content = match tool_result { + Ok(output) => { + let sanitized = + self.agent.safety().sanitize_tool_output(&tc.name, &output); + self.agent.safety().wrap_for_llm( + &tc.name, + &sanitized.content, + sanitized.was_modified, + ) + } + Err(e) => format!("Tool '{}' failed: {}", tc.name, e), + }; + + // Record sanitized result in thread + { + let mut sess = self.session.lock().await; + if let Some(thread) = sess.threads.get_mut(&self.thread_id) + && let Some(turn) = thread.last_turn_mut() + { + if is_tool_error { + turn.record_tool_error(result_content.clone()); + } else { + turn.record_tool_result(serde_json::json!(result_content)); + } + } + } + + reason_ctx.messages.push(ChatMessage::tool_result( + &tc.id, + &tc.name, + result_content, + )); + } + } + } + + // Return auth response after all results are recorded + if let Some(instructions) = deferred_auth { + return Ok(Some(LoopOutcome::Response(instructions))); + } + + // Handle approval if a tool needed it + if let Some((approval_idx, tc, tool)) = approval_needed { + let display_params = redact_params(&tc.arguments, tool.sensitive_params()); + let pending = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: tc.name.clone(), + parameters: tc.arguments.clone(), + display_parameters: display_params, + description: tool.description().to_string(), + tool_call_id: tc.id.clone(), + context_messages: reason_ctx.messages.clone(), + deferred_tool_calls: tool_calls[approval_idx + 1..].to_vec(), + user_timezone: Some(self.user_tz.name().to_string()), + }; + + return Ok(Some(LoopOutcome::NeedApproval(Box::new(pending)))); + } + + Ok(None) + } +} + /// Execute a chat tool without requiring `&Agent`. /// /// This standalone function enables parallel invocation from spawned JoinSet -/// tasks, which cannot borrow `&self`. It replicates the logic from -/// `Agent::execute_chat_tool`. +/// tasks, which cannot borrow `&self`. Delegates to the shared +/// `execute_tool_with_safety` pipeline. pub(super) async fn execute_chat_tool_standalone( tools: &crate::tools::ToolRegistry, safety: &crate::safety::SafetyLayer, @@ -875,91 +888,7 @@ pub(super) async fn execute_chat_tool_standalone( params: &serde_json::Value, job_ctx: &crate::context::JobContext, ) -> Result { - let tool = tools - .get(tool_name) - .await - .ok_or_else(|| crate::error::ToolError::NotFound { - name: tool_name.to_string(), - })?; - - // Validate tool parameters - let validation = safety.validator().validate_tool_params(params); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::>() - .join("; "); - return Err(crate::error::ToolError::InvalidParameters { - name: tool_name.to_string(), - reason: format!("Invalid tool parameters: {}", details), - } - .into()); - } - - let safe_params = redact_params(params, tool.sensitive_params()); - tracing::debug!( - tool = %tool_name, - params = %safe_params, - "Tool call started" - ); - - // Execute with per-tool timeout - let timeout = tool.execution_timeout(); - let start = std::time::Instant::now(); - let result = tokio::time::timeout(timeout, async { - tool.execute(params.clone(), job_ctx).await - }) - .await; - let elapsed = start.elapsed(); - - match &result { - Ok(Ok(output)) => { - let result_str = serde_json::to_string(&output.result) - .unwrap_or_else(|_| "".to_string()); - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - result = %result_str, - "Tool call succeeded" - ); - } - Ok(Err(e)) => { - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - error = %e, - "Tool call failed" - ); - } - Err(_) => { - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - timeout_secs = timeout.as_secs(), - "Tool call timed out" - ); - } - } - - let result = result - .map_err(|_| crate::error::ToolError::Timeout { - name: tool_name.to_string(), - timeout, - })? - .map_err(|e| crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: e.to_string(), - })?; - - serde_json::to_string_pretty(&result.result).map_err(|e| { - crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: format!("Failed to serialize result: {}", e), - } - .into() - }) + crate::tools::execute::execute_tool_with_safety(tools, safety, tool_name, params, job_ctx).await } /// Parsed auth result fields for emitting StatusUpdate::AuthRequired. diff --git a/src/agent/heartbeat.rs b/src/agent/heartbeat.rs index 4c05c1d5..15c51b61 100644 --- a/src/agent/heartbeat.rs +++ b/src/agent/heartbeat.rs @@ -189,7 +189,7 @@ impl HeartbeatRunner { // Skip during quiet hours if self.config.is_quiet_hours() { - tracing::debug!("Heartbeat skipped: quiet hours"); + tracing::trace!("Heartbeat skipped: quiet hours"); continue; } @@ -212,7 +212,7 @@ impl HeartbeatRunner { match self.check_heartbeat().await { HeartbeatResult::Ok => { - tracing::debug!("Heartbeat OK"); + tracing::trace!("Heartbeat OK"); self.consecutive_failures = 0; } HeartbeatResult::NeedsAttention(message) => { @@ -221,7 +221,7 @@ impl HeartbeatRunner { self.send_notification(&message).await; } HeartbeatResult::Skipped => { - tracing::debug!("Heartbeat skipped"); + tracing::trace!("Heartbeat skipped"); } HeartbeatResult::Failed(error) => { tracing::error!("Heartbeat failed: {}", error); @@ -303,7 +303,8 @@ impl HeartbeatRunner { .with_max_tokens(max_tokens) .with_temperature(0.3); - let reasoning = Reasoning::new(self.llm.clone()); + let reasoning = + Reasoning::new(self.llm.clone()).with_model_name(self.llm.active_model_name()); let (content, _usage) = match reasoning.complete(request).await { Ok(r) => r, Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)), diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 895a551a..de2434be 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -11,6 +11,7 @@ //! - Context compaction for long conversations mod agent_loop; +pub mod agentic_loop; mod attachments; mod commands; pub mod compaction; @@ -22,7 +23,7 @@ pub mod job_monitor; mod router; pub mod routine; pub mod routine_engine; -mod scheduler; +pub(crate) mod scheduler; mod self_repair; pub mod session; mod session_manager; @@ -30,8 +31,8 @@ pub mod submission; pub mod task; mod thread_ops; pub mod undo; -pub mod worker; +pub use crate::worker::{Worker, WorkerDeps}; pub(crate) use agent_loop::truncate_for_preview; pub use agent_loop::{Agent, AgentDeps}; pub use compaction::{CompactionResult, ContextCompactor}; @@ -47,4 +48,3 @@ pub use session_manager::SessionManager; pub use submission::{Submission, SubmissionParser, SubmissionResult}; pub use task::{Task, TaskContext, TaskHandler, TaskOutput}; pub use undo::{Checkpoint, UndoManager}; -pub use worker::{Worker, WorkerDeps}; diff --git a/src/agent/routine.rs b/src/agent/routine.rs index fdd61012..72226502 100644 --- a/src/agent/routine.rs +++ b/src/agent/routine.rs @@ -8,7 +8,7 @@ //! ┌──────────┐ ┌─────────┐ ┌──────────────────┐ //! │ Trigger │────▶│ Engine │────▶│ Execution Mode │ //! │ cron/event│ │guardrail│ │lightweight│full_job│ -//! │ webhook │ │ check │ └──────────────────┘ +//! │ system │ │ check │ └──────────────────┘ //! │ manual │ └─────────┘ │ //! └──────────┘ ▼ //! ┌──────────────┐ @@ -69,12 +69,15 @@ pub enum Trigger { /// Regex pattern to match against message content. pattern: String, }, - /// Fire on incoming webhook POST to /hooks/routine/{id}. - Webhook { - /// Optional webhook path suffix (defaults to routine id). - path: Option, - /// Optional shared secret for HMAC validation. - secret: Option, + /// Fire when a structured system event is emitted. + SystemEvent { + /// Event source namespace (e.g. "github", "workflow", "tool"). + source: String, + /// Event type within the source (e.g. "issue.opened"). + event_type: String, + /// Optional exact-match filters against payload top-level fields. + #[serde(default)] + filters: std::collections::HashMap, }, /// Only fires via tool call or CLI. Manual, @@ -86,7 +89,7 @@ impl Trigger { match self { Trigger::Cron { .. } => "cron", Trigger::Event { .. } => "event", - Trigger::Webhook { .. } => "webhook", + Trigger::SystemEvent { .. } => "system_event", Trigger::Manual => "manual", } } @@ -134,16 +137,39 @@ impl Trigger { .map(String::from); Ok(Trigger::Event { channel, pattern }) } - "webhook" => { - let path = config - .get("path") + "system_event" => { + let source = config + .get("source") .and_then(|v| v.as_str()) - .map(String::from); - let secret = config - .get("secret") + .ok_or_else(|| RoutineError::MissingField { + context: "system_event trigger".into(), + field: "source".into(), + })? + .to_string(); + let event_type = config + .get("event_type") .and_then(|v| v.as_str()) - .map(String::from); - Ok(Trigger::Webhook { path, secret }) + .ok_or_else(|| RoutineError::MissingField { + context: "system_event trigger".into(), + field: "event_type".into(), + })? + .to_string(); + let filters = config + .get("filters") + .and_then(|v| v.as_object()) + .map(|m| { + m.iter() + .filter_map(|(k, v)| { + json_value_as_filter_string(v).map(|s| (k.clone(), s)) + }) + .collect() + }) + .unwrap_or_default(); + Ok(Trigger::SystemEvent { + source, + event_type, + filters, + }) } "manual" => Ok(Trigger::Manual), other => Err(RoutineError::UnknownTriggerType { @@ -163,9 +189,14 @@ impl Trigger { "pattern": pattern, "channel": channel, }), - Trigger::Webhook { path, secret } => serde_json::json!({ - "path": path, - "secret": secret, + Trigger::SystemEvent { + source, + event_type, + filters, + } => serde_json::json!({ + "source": source, + "event_type": event_type, + "filters": filters, }), Trigger::Manual => serde_json::json!({}), } @@ -428,6 +459,19 @@ pub struct RoutineRun { pub created_at: DateTime, } +/// Convert a JSON value to a string for filter storage. +/// +/// Handles strings, numbers, and booleans — consistent with the matching +/// logic in `routine_engine::json_value_as_string`. +pub fn json_value_as_filter_string(v: &serde_json::Value) -> Option { + match v { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Number(n) => Some(n.to_string()), + serde_json::Value::Bool(b) => Some(b.to_string()), + _ => None, + } +} + /// Compute a content hash for event dedup. pub fn content_hash(content: &str) -> u64 { let mut hasher = DefaultHasher::new(); @@ -486,6 +530,24 @@ mod tests { if channel == Some("telegram".to_string()) && pattern == r"deploy\s+\w+")); } + #[test] + fn test_system_event_trigger_roundtrip() { + let mut filters = std::collections::HashMap::new(); + filters.insert("repo".to_string(), "nearai/ironclaw".to_string()); + filters.insert("action".to_string(), "opened".to_string()); + let trigger = Trigger::SystemEvent { + source: "github".to_string(), + event_type: "issue".to_string(), + filters: filters.clone(), + }; + let json = trigger.to_config_json(); + let parsed = Trigger::from_db("system_event", json).expect("parse system_event"); + assert!( + matches!(parsed, Trigger::SystemEvent { source, event_type, filters: f } + if source == "github" && event_type == "issue" && f == filters) + ); + } + #[test] fn test_action_lightweight_roundtrip() { let action = RoutineAction::Lightweight { @@ -623,12 +685,13 @@ mod tests { "event" ); assert_eq!( - Trigger::Webhook { - path: None, - secret: None + Trigger::SystemEvent { + source: String::new(), + event_type: String::new(), + filters: std::collections::HashMap::new(), } .type_tag(), - "webhook" + "system_event" ); assert_eq!(Trigger::Manual.type_tag(), "manual"); } diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index 3d27bdb1..b10021ef 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -32,9 +32,14 @@ use crate::llm::{ ChatMessage, CompletionRequest, FinishReason, LlmProvider, ToolCall, ToolCompletionRequest, }; use crate::safety::SafetyLayer; -use crate::tools::{ApprovalContext, ApprovalRequirement, ToolError, ToolRegistry, redact_params}; +use crate::tools::{ApprovalContext, ApprovalRequirement, ToolError, ToolRegistry}; use crate::workspace::Workspace; +enum EventMatcher { + Message { routine: Routine, regex: Regex }, + System { routine: Routine }, +} + /// The routine execution engine. pub struct RoutineEngine { config: RoutineConfig, @@ -45,8 +50,8 @@ pub struct RoutineEngine { notify_tx: mpsc::Sender, /// Currently running routine count (across all routines). running_count: Arc, - /// Compiled event regex cache: routine_id -> compiled regex. - event_cache: Arc>>, + /// Cached matchers for all event-driven routines. + event_cache: Arc>>, /// Scheduler for dispatching jobs (FullJob mode). scheduler: Option>, /// Tool registry for lightweight routine tool execution. @@ -87,9 +92,12 @@ impl RoutineEngine { Ok(routines) => { let mut cache = Vec::new(); for routine in routines { - if let Trigger::Event { ref pattern, .. } = routine.trigger { - match Regex::new(pattern) { - Ok(re) => cache.push((routine.id, routine.clone(), re)), + match &routine.trigger { + Trigger::Event { pattern, .. } => match Regex::new(pattern) { + Ok(re) => cache.push(EventMatcher::Message { + routine: routine.clone(), + regex: re, + }), Err(e) => { tracing::warn!( routine = %routine.name, @@ -97,12 +105,18 @@ impl RoutineEngine { pattern, e ); } + }, + Trigger::SystemEvent { .. } => { + cache.push(EventMatcher::System { + routine: routine.clone(), + }); } + _ => {} } } let count = cache.len(); *self.event_cache.write().await = cache; - tracing::debug!("Refreshed event cache: {} routines", count); + tracing::trace!("Refreshed event cache: {} routines", count); } Err(e) => { tracing::error!("Failed to refresh event cache: {}", e); @@ -118,7 +132,11 @@ impl RoutineEngine { let cache = self.event_cache.read().await; let mut fired = 0; - for (_, routine, re) in cache.iter() { + for matcher in cache.iter() { + let (routine, re) = match matcher { + EventMatcher::Message { routine, regex } => (routine, regex), + EventMatcher::System { .. } => continue, + }; // Channel filter if let Trigger::Event { channel: Some(ch), .. @@ -135,13 +153,13 @@ impl RoutineEngine { // Cooldown check if !self.check_cooldown(routine) { - tracing::debug!(routine = %routine.name, "Skipped: cooldown active"); + tracing::trace!(routine = %routine.name, "Skipped: cooldown active"); continue; } // Concurrent run check if !self.check_concurrent(routine).await { - tracing::debug!(routine = %routine.name, "Skipped: max concurrent reached"); + tracing::trace!(routine = %routine.name, "Skipped: max concurrent reached"); continue; } @@ -159,6 +177,88 @@ impl RoutineEngine { fired } + /// Emit a structured event to system-event routines. + /// + /// Returns the number of routines that were fired. + pub async fn emit_system_event( + &self, + source: &str, + event_type: &str, + payload: &serde_json::Value, + user_id: Option<&str>, + ) -> usize { + let cache = self.event_cache.read().await; + let mut fired = 0; + + for matcher in cache.iter() { + let routine = match matcher { + EventMatcher::System { routine } => routine, + EventMatcher::Message { .. } => continue, + }; + + let Trigger::SystemEvent { + source: expected_source, + event_type: expected_event, + filters, + } = &routine.trigger + else { + continue; + }; + + if !expected_source.eq_ignore_ascii_case(source) + || !expected_event.eq_ignore_ascii_case(event_type) + { + continue; + } + + if let Some(uid) = user_id + && routine.user_id != uid + { + continue; + } + + let mut matched = true; + for (key, expected) in filters { + let Some(actual) = payload + .get(key) + .and_then(crate::agent::routine::json_value_as_filter_string) + else { + tracing::debug!(routine = %routine.name, filter_key = %key, "Filter key not found in payload"); + matched = false; + break; + }; + if !actual.eq_ignore_ascii_case(expected) { + matched = false; + break; + } + } + if !matched { + continue; + } + + if !self.check_cooldown(routine) { + tracing::debug!(routine = %routine.name, "Skipped: cooldown active"); + continue; + } + + if !self.check_concurrent(routine).await { + tracing::debug!(routine = %routine.name, "Skipped: max concurrent reached"); + continue; + } + + if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines { + tracing::warn!(routine = %routine.name, "Skipped: global max concurrent reached"); + continue; + } + + let detail = truncate(&format!("{source}:{event_type}"), 200); + self.spawn_fire(routine.clone(), "system_event", Some(detail)); + fired += 1; + } + + fired + } + /// Check all due cron routines and fire them. Called by the cron ticker. pub async fn check_cron_triggers(&self) { let routines = match self.store.list_due_cron_routines().await { @@ -913,13 +1013,6 @@ async fn execute_routine_tool( return Err(format!("Invalid tool parameters: {}", details).into()); } - let safe_params = redact_params(&tc.arguments, tool.sensitive_params()); - tracing::debug!( - tool = %tc.name, - params = %safe_params, - "Lightweight routine tool call started" - ); - // Execute with per-tool timeout let timeout = tool.execution_timeout(); let start = std::time::Instant::now(); @@ -929,12 +1022,14 @@ async fn execute_routine_tool( .await; let elapsed = start.elapsed(); + // Log tool execution result (single consolidated log) match &result { Ok(Ok(_)) => { tracing::debug!( tool = %tc.name, elapsed_ms = elapsed.as_millis() as u64, - "Lightweight routine tool call succeeded" + status = "succeeded", + "Lightweight routine tool execution completed" ); } Ok(Err(e)) => { @@ -942,7 +1037,8 @@ async fn execute_routine_tool( tool = %tc.name, elapsed_ms = elapsed.as_millis() as u64, error = %e, - "Lightweight routine tool call failed" + status = "failed", + "Lightweight routine tool execution completed" ); } Err(_) => { @@ -950,7 +1046,8 @@ async fn execute_routine_tool( tool = %tc.name, elapsed_ms = elapsed.as_millis() as u64, timeout_secs = timeout.as_secs(), - "Lightweight routine tool call timed out" + status = "timeout", + "Lightweight routine tool execution completed" ); } } diff --git a/src/agent/scheduler.rs b/src/agent/scheduler.rs index 85f3f6eb..5e4bf01a 100644 --- a/src/agent/scheduler.rs +++ b/src/agent/scheduler.rs @@ -9,7 +9,6 @@ use tokio::task::JoinHandle; use uuid::Uuid; use crate::agent::task::{Task, TaskContext, TaskOutput}; -use crate::agent::worker::{Worker, WorkerDeps}; use crate::channels::web::types::SseEvent; use crate::config::AgentConfig; use crate::context::{ContextManager, JobContext, JobState}; @@ -19,6 +18,7 @@ use crate::hooks::HookRegistry; use crate::llm::LlmProvider; use crate::safety::SafetyLayer; use crate::tools::{ApprovalContext, ToolRegistry}; +use crate::worker::job::{Worker, WorkerDeps}; /// Message to send to a worker. #[derive(Debug)] @@ -160,24 +160,36 @@ impl Scheduler { .create_job_for_user(user_id, title, description) .await?; - // Apply token budget from config, allowing per-job metadata override. - let max_tokens = metadata + // Apply metadata and token budget in a single atomic update. + // This prevents concurrent workers from observing partial state. + // Cap user-supplied max_tokens at the configured limit (Issue #815). + let user_max_tokens = metadata .as_ref() .and_then(|m| m.get("max_tokens")) - .and_then(|v| v.as_u64()) + .and_then(|v| v.as_u64()); + + let max_tokens = user_max_tokens + .map(|user_val| { + if self.config.max_tokens_per_job == 0 { + // Config is "unlimited": use the user-supplied value directly. + user_val + } else { + std::cmp::min(user_val, self.config.max_tokens_per_job) + } + }) .unwrap_or(self.config.max_tokens_per_job); - // Apply metadata if provided + // Apply both metadata and token budget in one closure (Issue #813: atomic update) if let Some(meta) = metadata { self.context_manager .update_context(job_id, |ctx| { ctx.metadata = meta; + if max_tokens > 0 { + ctx.max_tokens = max_tokens; + } }) .await?; - } - - // Set token budget (separate update to avoid overwriting metadata) - if max_tokens > 0 { + } else if max_tokens > 0 { self.context_manager .update_context(job_id, |ctx| { ctx.max_tokens = max_tokens; @@ -462,6 +474,9 @@ impl Scheduler { } /// Execute a single tool as a subtask. + /// + /// Performs scheduler-specific checks (approval, cancellation) then + /// delegates to the shared `execute_tool_with_safety` pipeline. async fn execute_tool_task( tools: Arc, context_manager: Arc, @@ -473,7 +488,7 @@ impl Scheduler { ) -> Result { let start = std::time::Instant::now(); - // Get the tool + // Get the tool for approval check let tool = tools.get(tool_name).await.ok_or_else(|| { Error::Tool(crate::error::ToolError::NotFound { name: tool_name.to_string(), @@ -490,6 +505,7 @@ impl Scheduler { .into()); } + // Scheduler-specific approval check let requirement = tool.requires_approval(¶ms); let blocked = ApprovalContext::is_blocked_or_default(&approval_context, tool_name, requirement); @@ -500,41 +516,23 @@ impl Scheduler { .into()); } - // Validate tool parameters - let validation = safety.validator().validate_tool_params(¶ms); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::>() - .join("; "); - return Err(crate::error::ToolError::InvalidParameters { + // Delegate to shared tool execution pipeline + let output_str = crate::tools::execute::execute_tool_with_safety( + &tools, &safety, tool_name, ¶ms, &job_ctx, + ) + .await?; + + // Parse back to Value for TaskOutput; this should be infallible given + // `execute_tool_with_safety` uses `serde_json::to_string_pretty`, but if it + // ever fails we surface a clear error instead of silently changing types. + let result_value: serde_json::Value = serde_json::from_str(&output_str).map_err(|e| { + Error::Tool(crate::error::ToolError::ExecutionFailed { name: tool_name.to_string(), - reason: format!("Invalid tool parameters: {}", details), - } - .into()); - } + reason: format!("Failed to parse tool output as JSON: {}", e), + }) + })?; - // Execute with per-tool timeout - let tool_timeout = tool.execution_timeout(); - let result = - tokio::time::timeout(tool_timeout, async { tool.execute(params, &job_ctx).await }) - .await - .map_err(|_| { - Error::Tool(crate::error::ToolError::Timeout { - name: tool_name.to_string(), - timeout: tool_timeout, - }) - })? - .map_err(|e| { - Error::Tool(crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: e.to_string(), - }) - })?; - - Ok(TaskOutput::new(result.result, start.elapsed())) + Ok(TaskOutput::new(result_value, start.elapsed())) } /// Stop a running job. @@ -699,8 +697,140 @@ impl Scheduler { mod tests { use super::*; use crate::config::SafetyConfig; + use crate::llm::{ + CompletionRequest, CompletionResponse, LlmError, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, + }; use crate::safety::SafetyLayer; use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput}; + use rust_decimal_macros::dec; + + /// Minimal LLM provider stub for scheduler tests that don't exercise LLM calls. + struct StubLlm; + + #[async_trait::async_trait] + impl LlmProvider for StubLlm { + fn model_name(&self) -> &str { + "stub" + } + fn cost_per_token(&self) -> (rust_decimal::Decimal, rust_decimal::Decimal) { + (dec!(0), dec!(0)) + } + async fn complete(&self, _req: CompletionRequest) -> Result { + Err(LlmError::RequestFailed { + provider: "stub".into(), + reason: "not implemented".into(), + }) + } + async fn complete_with_tools( + &self, + _req: ToolCompletionRequest, + ) -> Result { + Err(LlmError::RequestFailed { + provider: "stub".into(), + reason: "not implemented".into(), + }) + } + } + + /// Create a Scheduler for token-budget tests. The LLM stub will fail if a + /// worker actually tries to call it, but `dispatch_job` sets the token + /// budget *before* spawning the worker so we can inspect the context + /// immediately after dispatch. + fn make_test_scheduler(max_tokens_per_job: u64) -> Scheduler { + let config = AgentConfig { + name: "test".to_string(), + max_parallel_jobs: 5, + job_timeout: std::time::Duration::from_secs(30), + stuck_threshold: std::time::Duration::from_secs(300), + repair_check_interval: std::time::Duration::from_secs(3600), + max_repair_attempts: 0, + use_planning: false, + session_idle_timeout: std::time::Duration::from_secs(3600), + allow_local_tools: true, + max_cost_per_day_cents: None, + max_actions_per_hour: None, + max_tool_iterations: 10, + auto_approve_tools: true, + default_timezone: "UTC".to_string(), + max_tokens_per_job, + }; + let cm = Arc::new(ContextManager::new(5)); + let llm: Arc = Arc::new(StubLlm); + let safety = Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })); + let tools = Arc::new(ToolRegistry::new()); + let hooks = Arc::new(HookRegistry::default()); + + Scheduler::new(config, cm, llm, safety, tools, None, hooks) + } + + #[tokio::test] + async fn test_dispatch_job_caps_user_max_tokens() { + let sched = make_test_scheduler(1000); + let meta = serde_json::json!({ "max_tokens": 5000 }); + let job_id = sched + .dispatch_job("user1", "test", "desc", Some(meta)) + .await + .unwrap(); + + let ctx = sched.context_manager.get_context(job_id).await.unwrap(); + assert_eq!(ctx.max_tokens, 1000, "should cap at configured limit"); + } + + #[tokio::test] + async fn test_dispatch_job_unlimited_config_preserves_user_tokens() { + let sched = make_test_scheduler(0); // 0 = unlimited + let meta = serde_json::json!({ "max_tokens": 5000 }); + let job_id = sched + .dispatch_job("user1", "test", "desc", Some(meta)) + .await + .unwrap(); + + let ctx = sched.context_manager.get_context(job_id).await.unwrap(); + assert_eq!( + ctx.max_tokens, 5000, + "unlimited config should preserve user value" + ); + } + + #[tokio::test] + async fn test_dispatch_job_no_user_tokens_uses_config() { + let sched = make_test_scheduler(2000); + let job_id = sched + .dispatch_job("user1", "test", "desc", None) + .await + .unwrap(); + + let ctx = sched.context_manager.get_context(job_id).await.unwrap(); + assert_eq!( + ctx.max_tokens, 2000, + "should use config default when no user value" + ); + } + + #[tokio::test] + async fn test_dispatch_job_atomic_metadata_and_tokens() { + let sched = make_test_scheduler(10_000); + let meta = serde_json::json!({ + "max_tokens": 3000, + "custom_key": "custom_value" + }); + let job_id = sched + .dispatch_job("user1", "test", "desc", Some(meta)) + .await + .unwrap(); + + let ctx = sched.context_manager.get_context(job_id).await.unwrap(); + assert_eq!(ctx.max_tokens, 3000, "should use user value within limit"); + assert_eq!( + ctx.metadata.get("custom_key").and_then(|v| v.as_str()), + Some("custom_value"), + "metadata should be set atomically with token budget" + ); + } #[test] fn test_scheduler_creation() { diff --git a/src/agent/self_repair.rs b/src/agent/self_repair.rs index 5ac8e8aa..a67fe23e 100644 --- a/src/agent/self_repair.rs +++ b/src/agent/self_repair.rs @@ -334,22 +334,21 @@ impl RepairTask { // Check for stuck jobs let stuck_jobs = self.repair.detect_stuck_jobs().await; for job in stuck_jobs { - tracing::info!("Attempting to repair stuck job {}", job.job_id); match self.repair.repair_stuck_job(&job).await { Ok(RepairResult::Success { message }) => { - tracing::info!("Repair succeeded: {}", message); + tracing::info!(job = %job.job_id, status = "success", "Stuck job repair completed: {}", message); } Ok(RepairResult::Retry { message }) => { - tracing::warn!("Repair needs retry: {}", message); + tracing::debug!(job = %job.job_id, status = "retry", "Stuck job repair needs retry: {}", message); } Ok(RepairResult::Failed { message }) => { - tracing::error!("Repair failed: {}", message); + tracing::error!(job = %job.job_id, status = "failed", "Stuck job repair failed: {}", message); } Ok(RepairResult::ManualRequired { message }) => { - tracing::warn!("Manual intervention needed: {}", message); + tracing::warn!(job = %job.job_id, status = "manual", "Stuck job repair requires manual intervention: {}", message); } Err(e) => { - tracing::error!("Repair error: {}", e); + tracing::error!(job = %job.job_id, "Stuck job repair error: {}", e); } } } @@ -357,13 +356,12 @@ impl RepairTask { // Check for broken tools let broken_tools = self.repair.detect_broken_tools().await; for tool in broken_tools { - tracing::info!("Attempting to repair broken tool: {}", tool.name); match self.repair.repair_broken_tool(&tool).await { Ok(result) => { - tracing::info!("Tool repair result: {:?}", result); + tracing::debug!(tool = %tool.name, status = "completed", "Tool repair completed: {:?}", result); } Err(e) => { - tracing::error!("Tool repair error: {}", e); + tracing::error!(tool = %tool.name, "Tool repair error: {}", e); } } } diff --git a/src/agent/thread_ops.rs b/src/agent/thread_ops.rs index 758e98ed..e7f526e3 100644 --- a/src/agent/thread_ops.rs +++ b/src/agent/thread_ops.rs @@ -113,6 +113,13 @@ impl Agent { thread_id: Uuid, content: &str, ) -> Result { + tracing::debug!( + message_id = %message.id, + thread_id = %thread_id, + content_len = content.len(), + "Processing user input" + ); + // First check thread state without holding lock during I/O let thread_state = { let sess = session.lock().await; @@ -123,19 +130,41 @@ impl Agent { thread.state }; + tracing::debug!( + message_id = %message.id, + thread_id = %thread_id, + thread_state = ?thread_state, + "Checked thread state" + ); + // Check thread state match thread_state { ThreadState::Processing => { + tracing::warn!( + message_id = %message.id, + thread_id = %thread_id, + "Thread is processing, rejecting new input" + ); return Ok(SubmissionResult::error( "Turn in progress. Use /interrupt to cancel.", )); } ThreadState::AwaitingApproval => { + tracing::warn!( + message_id = %message.id, + thread_id = %thread_id, + "Thread awaiting approval, rejecting new input" + ); return Ok(SubmissionResult::error( "Waiting for approval. Use /interrupt to cancel.", )); } ThreadState::Completed => { + tracing::warn!( + message_id = %message.id, + thread_id = %thread_id, + "Thread completed, rejecting new input" + ); return Ok(SubmissionResult::error( "Thread completed. Use /thread new.", )); @@ -269,9 +298,20 @@ impl Agent { }; // Persist user message to DB immediately so it survives crashes + tracing::debug!( + message_id = %message.id, + thread_id = %thread_id, + "Persisting user message to DB" + ); self.persist_user_message(thread_id, &message.user_id, effective_content) .await; + tracing::debug!( + message_id = %message.id, + thread_id = %thread_id, + "User message persisted, starting agentic loop" + ); + // Send thinking status let _ = self .channels @@ -812,19 +852,12 @@ impl Agent { // Sanitize tool result, then record the cleaned version in the // thread. Must happen before auth intercept check which may return early. let is_tool_error = tool_result.is_err(); - let result_content = match &tool_result { - Ok(output) => { - let sanitized = self - .safety() - .sanitize_tool_output(&pending.tool_name, output); - self.safety().wrap_for_llm( - &pending.tool_name, - &sanitized.content, - sanitized.was_modified, - ) - } - Err(e) => format!("Error: {}", e), - }; + let (result_content, _) = crate::tools::execute::process_tool_result( + self.safety(), + &pending.tool_name, + &pending.tool_call_id, + &tool_result, + ); // Record sanitized result in thread { @@ -1064,17 +1097,12 @@ impl Agent { // Sanitize first, then record the cleaned version in thread. // Must happen before auth detection which may set deferred_auth. let is_deferred_error = deferred_result.is_err(); - let deferred_content = match &deferred_result { - Ok(output) => { - let sanitized = self.safety().sanitize_tool_output(&tc.name, output); - self.safety().wrap_for_llm( - &tc.name, - &sanitized.content, - sanitized.was_modified, - ) - } - Err(e) => format!("Error: {}", e), - }; + let (deferred_content, _) = crate::tools::execute::process_tool_result( + self.safety(), + &tc.name, + &tc.id, + &deferred_result, + ); // Record sanitized result in thread { diff --git a/src/channels/channel.rs b/src/channels/channel.rs index e126ca1f..938b1f4f 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -344,9 +344,28 @@ pub trait Channel: Send + Sync { } } +/// Trait for channels that support hot-secret-swapping during SIGHUP reload. +/// +/// This allows channels to update authentication credentials without restarting, +/// enabling zero-downtime configuration reloads. Channels that don't support +/// secret updates can simply not implement this trait. +#[async_trait] +pub trait ChannelSecretUpdater: Send + Sync { + /// Update the secret for this channel. + /// + /// Called during SIGHUP configuration reload. Implementation should: + /// - Apply the new secret atomically + /// - Not fail the entire reload if secret update fails + /// - Log appropriate errors/info messages + /// + /// The secret is optional (may be None if secret is no longer configured). + async fn update_secret(&self, new_secret: Option); +} + #[cfg(test)] mod tests { use super::*; + use crate::testing::credentials::TEST_REDACT_SECRET_123; /// Stub tool that marks `"value"` as sensitive. struct SecretTool; @@ -376,7 +395,7 @@ mod tests { #[test] fn tool_completed_redacts_sensitive_params_on_failure() { - let params = serde_json::json!({"name": "api_key", "value": "sk-secret-123"}); + let params = serde_json::json!({"name": "api_key", "value": TEST_REDACT_SECRET_123}); let err: Result = Err(crate::error::ToolError::ExecutionFailed { name: "secret_save".into(), @@ -411,7 +430,7 @@ mod tests { param_str ); assert!( - !param_str.contains("sk-secret-123"), + !param_str.contains(TEST_REDACT_SECRET_123), "raw secret should not appear: {}", param_str ); diff --git a/src/channels/http.rs b/src/channels/http.rs index 74799b04..e40e251b 100644 --- a/src/channels/http.rs +++ b/src/channels/http.rs @@ -10,7 +10,7 @@ use axum::{ response::IntoResponse, routing::{get, post}, }; -use secrecy::ExposeSecret; +use secrecy::{ExposeSecret, SecretString}; use serde::{Deserialize, Serialize}; use subtle::ConstantTimeEq; use tokio::sync::{RwLock, mpsc, oneshot}; @@ -18,7 +18,8 @@ use tokio_stream::wrappers::ReceiverStream; use uuid::Uuid; use crate::channels::{ - AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse, + AttachmentKind, Channel, ChannelSecretUpdater, IncomingAttachment, IncomingMessage, + MessageStream, OutgoingResponse, }; use crate::config::HttpConfig; use crate::error::ChannelError; @@ -29,13 +30,16 @@ pub struct HttpChannel { state: Arc, } -struct HttpChannelState { +pub struct HttpChannelState { /// Sender for incoming messages. tx: RwLock>>, /// Pending responses keyed by message ID. pending_responses: RwLock>>, /// Expected webhook secret for authentication (if configured). - webhook_secret: Option, + /// Stored in a separate Arc> to avoid contending with other state operations. + /// Rarely changes (only on SIGHUP), so isolated from hot-path state accesses. + /// Uses SecretString to prevent accidental logging and memory dump exposure. + webhook_secret: Arc>>, /// Fixed user ID for this HTTP channel. user_id: String, /// Rate limiting state. @@ -48,6 +52,14 @@ struct RateLimitState { request_count: u32, } +impl HttpChannelState { + /// Update the webhook secret in-place without restarting the listener. + /// Called during SIGHUP to hot-swap credentials. + pub async fn update_secret(&self, new_secret: Option) { + *self.webhook_secret.write().await = new_secret; + } +} + /// Maximum JSON body size for webhook requests (15 MB, to support base64 image attachments /// with ~33% overhead from base64 encoding). const MAX_BODY_BYTES: usize = 15 * 1024 * 1024; @@ -67,7 +79,7 @@ impl HttpChannel { let webhook_secret = config .webhook_secret .as_ref() - .map(|s| s.expose_secret().to_string()); + .map(|s| SecretString::from(s.expose_secret().to_string())); let user_id = config.user_id.clone(); Self { @@ -75,7 +87,7 @@ impl HttpChannel { state: Arc::new(HttpChannelState { tx: RwLock::new(None), pending_responses: RwLock::new(std::collections::HashMap::new()), - webhook_secret, + webhook_secret: Arc::new(RwLock::new(webhook_secret)), user_id, rate_limit: tokio::sync::Mutex::new(RateLimitState { window_start: std::time::Instant::now(), @@ -102,6 +114,16 @@ impl HttpChannel { pub fn addr(&self) -> (&str, u16) { (&self.config.host, self.config.port) } + + /// Return a shared handle to the channel state for out-of-band updates. + pub fn shared_state(&self) -> Arc { + Arc::clone(&self.state) + } + + /// Update the webhook secret in-place without restarting the listener. + pub async fn update_secret(&self, new_secret: Option) { + self.state.update_secret(new_secret).await; + } } #[derive(Debug, Deserialize)] @@ -201,9 +223,10 @@ async fn webhook_handler( }); // Validate secret if configured - if let Some(ref expected_secret) = state.webhook_secret { + if let Some(ref expected_secret) = *state.webhook_secret.read().await { + let expected_bytes = expected_secret.expose_secret().as_bytes(); match &req.secret { - Some(provided) if bool::from(provided.as_bytes().ct_eq(expected_secret.as_bytes())) => { + Some(provided) if bool::from(provided.as_bytes().ct_eq(expected_bytes)) => { // Secret matches, continue } Some(_) => { @@ -372,9 +395,14 @@ async fn process_message( None }; - // Send message to the channel - let tx_guard = state.tx.read().await; - if let Some(tx) = tx_guard.as_ref() { + // Clone sender while holding read lock, then release lock before async send. + // This prevents blocking other webhook handlers during the async I/O. + let tx = { + let guard = state.tx.read().await; + guard.as_ref().cloned() + }; + + if let Some(tx) = tx { if tx.send(msg).await.is_err() { return ( StatusCode::INTERNAL_SERVER_ERROR, @@ -395,7 +423,6 @@ async fn process_message( }), ); } - drop(tx_guard); // Wait for response if requested let response = if let Some(rx) = response_rx { @@ -428,7 +455,7 @@ impl Channel for HttpChannel { } async fn start(&self) -> Result { - if self.state.webhook_secret.is_none() { + if self.state.webhook_secret.read().await.is_none() { return Err(ChannelError::StartupFailed { name: "http".to_string(), reason: "HTTP webhook secret is required (set HTTP_WEBHOOK_SECRET)".to_string(), @@ -475,6 +502,16 @@ impl Channel for HttpChannel { } } +/// Implement secret update for HTTP channel state. +/// This allows SIGHUP handler to update secrets generically via the trait. +#[async_trait] +impl ChannelSecretUpdater for HttpChannelState { + async fn update_secret(&self, new_secret: Option) { + *self.webhook_secret.write().await = new_secret; + tracing::info!("HTTP webhook secret updated"); + } +} + #[cfg(test)] mod tests { use axum::body::Body; @@ -562,4 +599,156 @@ mod tests { let resp = app.oneshot(req).await.unwrap(); assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); } + + #[tokio::test] + async fn test_update_secret_hot_swap() { + let channel = test_channel(Some("old-secret")); + let _stream = channel.start().await.unwrap(); + let app1 = channel.routes(); + + // Request with old-secret should succeed + let body_old = serde_json::json!({ + "content": "hello", + "secret": "old-secret" + }); + let req1 = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body_old).unwrap())) + .unwrap(); + let resp1 = app1.oneshot(req1).await.unwrap(); + assert_eq!( + resp1.status(), + StatusCode::OK, + "old secret should work initially" + ); + + // Update secret to new-secret + channel + .update_secret(Some(SecretString::from("new-secret".to_string()))) + .await; + + let app2 = channel.routes(); + + // Request with old-secret should fail + let req2 = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body_old).unwrap())) + .unwrap(); + let resp2 = app2.oneshot(req2).await.unwrap(); + assert_eq!( + resp2.status(), + StatusCode::UNAUTHORIZED, + "old secret should fail after update" + ); + + let app3 = channel.routes(); + + // Request with new-secret should succeed + let body_new = serde_json::json!({ + "content": "hello", + "secret": "new-secret" + }); + let req3 = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body_new).unwrap())) + .unwrap(); + let resp3 = app3.oneshot(req3).await.unwrap(); + assert_eq!( + resp3.status(), + StatusCode::OK, + "new secret should work after update" + ); + } + + #[tokio::test] + async fn test_concurrent_requests_during_secret_update() { + use std::sync::Arc as StdArc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + let channel = test_channel(Some("initial-secret")); + let _stream = channel.start().await.unwrap(); + let app = channel.routes(); + + // Counters for request outcomes + let success_count = StdArc::new(AtomicUsize::new(0)); + + let mut handles = vec![]; + + // Spawn 5 concurrent tasks that keep making requests with the initial secret + for i in 0..5 { + let app = app.clone(); + let success = StdArc::clone(&success_count); + + let handle = tokio::spawn(async move { + let body = serde_json::json!({ + "content": format!("test-{}", i), + "secret": "initial-secret" + }); + + let req = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + if resp.status() == StatusCode::OK { + success.fetch_add(1, Ordering::SeqCst); + } + }); + handles.push(handle); + } + + // Update secret mid-flight (tests that RwLock allows readers while writer holds lock) + tokio::time::sleep(Duration::from_millis(5)).await; + channel + .update_secret(Some(SecretString::from("updated-secret".to_string()))) + .await; + + // Spawn 5 more tasks that use the new secret + for i in 5..10 { + let app = app.clone(); + let success = StdArc::clone(&success_count); + + let handle = tokio::spawn(async move { + let body = serde_json::json!({ + "content": format!("test-{}", i), + "secret": "updated-secret" + }); + + let req = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + if resp.status() == StatusCode::OK { + success.fetch_add(1, Ordering::SeqCst); + } + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + let _ = handle.await; + } + + // Verify all requests succeeded with their respective secrets + assert_eq!( + success_count.load(Ordering::SeqCst), + 10, + "All concurrent requests should succeed with correct secrets after update" + ); + } } diff --git a/src/channels/mod.rs b/src/channels/mod.rs index 095c96c1..038b432f 100644 --- a/src/channels/mod.rs +++ b/src/channels/mod.rs @@ -37,10 +37,10 @@ pub mod web; mod webhook_server; pub use channel::{ - AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse, - StatusUpdate, + AttachmentKind, Channel, ChannelSecretUpdater, IncomingAttachment, IncomingMessage, + MessageStream, OutgoingResponse, StatusUpdate, }; -pub use http::HttpChannel; +pub use http::{HttpChannel, HttpChannelState}; pub use manager::ChannelManager; pub use repl::ReplChannel; pub use signal::SignalChannel; diff --git a/src/channels/wasm/setup.rs b/src/channels/wasm/setup.rs index ca202b3b..cf448750 100644 --- a/src/channels/wasm/setup.rs +++ b/src/channels/wasm/setup.rs @@ -218,25 +218,31 @@ async fn register_channel( } // Inject credentials from secrets store / environment. - if let Some(secrets) = secrets_store { - match inject_channel_credentials(&channel_arc, secrets.as_ref(), &channel_name).await { - Ok(count) => { - if count > 0 { - tracing::info!( - channel = %channel_name, - credentials_injected = count, - "Channel credentials injected" - ); - } - } - Err(e) => { - tracing::error!( + match inject_channel_credentials( + &channel_arc, + secrets_store + .as_ref() + .map(|s| s.as_ref() as &dyn SecretsStore), + &channel_name, + ) + .await + { + Ok(count) => { + if count > 0 { + tracing::info!( channel = %channel_name, - error = %e, - "Failed to inject channel credentials" + credentials_injected = count, + "Channel credentials injected" ); } } + Err(e) => { + tracing::error!( + channel = %channel_name, + error = %e, + "Failed to inject channel credentials" + ); + } } (channel_name, Box::new(SharedWasmChannel::new(channel_arc))) @@ -247,58 +253,70 @@ async fn register_channel( /// Looks for secrets matching the pattern `{channel_name}_*` and injects them /// as credential placeholders (e.g., `telegram_bot_token` -> `{TELEGRAM_BOT_TOKEN}`). /// -/// Falls back to environment variables with the uppercase name if not found -/// in the secrets store (e.g., `TELEGRAM_BOT_TOKEN`). +/// Falls back to environment variables starting with the uppercase channel name +/// prefix (e.g., `TELEGRAM_` for channel `telegram`) for missing credentials. +/// +/// Returns the number of credentials injected. pub async fn inject_channel_credentials( channel: &Arc, - secrets: &dyn SecretsStore, + secrets: Option<&dyn SecretsStore>, channel_name: &str, ) -> anyhow::Result { - let all_secrets = secrets - .list("default") - .await - .map_err(|e| anyhow::anyhow!("Failed to list secrets: {}", e))?; + if channel_name.trim().is_empty() { + return Ok(0); + } - let prefix = format!("{}_", channel_name); let mut count = 0; let mut injected_placeholders = HashSet::new(); - for secret_meta in all_secrets { - if !secret_meta.name.starts_with(&prefix) { - continue; - } + // 1. Try injecting from persistent secrets store if available + if let Some(secrets) = secrets { + let all_secrets = secrets + .list("default") + .await + .map_err(|e| anyhow::anyhow!("Failed to list secrets: {}", e))?; - let decrypted = match secrets.get_decrypted("default", &secret_meta.name).await { - Ok(d) => d, - Err(e) => { - tracing::warn!( - secret = %secret_meta.name, - error = %e, - "Failed to decrypt secret for channel credential injection" - ); + let prefix = format!("{}_", channel_name.to_ascii_lowercase()); + + for secret_meta in all_secrets { + if !secret_meta.name.to_ascii_lowercase().starts_with(&prefix) { continue; } - }; - let placeholder = secret_meta.name.to_uppercase(); + let decrypted = match secrets.get_decrypted("default", &secret_meta.name).await { + Ok(d) => d, + Err(e) => { + tracing::warn!( + secret = %secret_meta.name, + error = %e, + "Failed to decrypt secret for channel credential injection" + ); + continue; + } + }; - tracing::debug!( - channel = %channel_name, - secret = %secret_meta.name, - placeholder = %placeholder, - "Injecting credential" - ); + let placeholder = secret_meta.name.to_uppercase(); - channel - .set_credential(&placeholder, decrypted.expose().to_string()) - .await; - injected_placeholders.insert(placeholder); - count += 1; + tracing::debug!( + channel = %channel_name, + secret = %secret_meta.name, + placeholder = %placeholder, + "Injecting credential" + ); + + channel + .set_credential(&placeholder, decrypted.expose().to_string()) + .await; + injected_placeholders.insert(placeholder); + count += 1; + } } - // Fall back to environment variables for required secrets not found in the store. - // This allows channels to work when configured via env vars (e.g., TELEGRAM_BOT_TOKEN) - // without requiring the setup wizard to have run. + // 2. Fall back to environment variables for credentials not in the secrets store. + // Only env vars starting with the channel's uppercase prefix are allowed + // (e.g., TELEGRAM_ for channel "telegram") to prevent reading unrelated host + // credentials like AWS_SECRET_ACCESS_KEY. + let prefix = format!("{}_", channel_name.to_ascii_uppercase()); let caps = channel.capabilities(); if let Some(ref http_cap) = caps.tool_capabilities.http { for cred_mapping in http_cap.credentials.values() { @@ -306,6 +324,14 @@ pub async fn inject_channel_credentials( if injected_placeholders.contains(&placeholder) { continue; } + if !placeholder.starts_with(&prefix) { + tracing::warn!( + channel = %channel_name, + placeholder = %placeholder, + "Ignoring non-prefixed credential placeholder in environment fallback" + ); + continue; + } if let Ok(env_value) = std::env::var(&placeholder) && !env_value.is_empty() { diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 3b788e89..a9fa4dbf 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -3059,6 +3059,7 @@ mod tests { }; use crate::channels::wasm::wrapper::{HttpResponse, WasmChannel}; use crate::pairing::PairingStore; + use crate::testing::credentials::TEST_TELEGRAM_BOT_TOKEN; use crate::tools::wasm::ResourceLimits; fn create_test_channel() -> WasmChannel { @@ -4009,7 +4010,7 @@ mod tests { let mut creds = std::collections::HashMap::new(); creds.insert( "TELEGRAM_BOT_TOKEN".to_string(), - "8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis".to_string(), + TEST_TELEGRAM_BOT_TOKEN.to_string(), ); creds.insert("OTHER_SECRET".to_string(), "s3cret".to_string()); @@ -4022,13 +4023,15 @@ mod tests { Arc::new(PairingStore::new()), ); - let error = "HTTP request failed: error sending request for url \ - (https://api.telegram.org/bot8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis/getUpdates)"; + let error = format!( + "HTTP request failed: error sending request for url \ + (https://api.telegram.org/bot{TEST_TELEGRAM_BOT_TOKEN}/getUpdates)" + ); - let redacted = store.redact_credentials(error); + let redacted = store.redact_credentials(&error); assert!( - !redacted.contains("8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis"), + !redacted.contains(TEST_TELEGRAM_BOT_TOKEN), "credential value should be redacted" ); assert!( diff --git a/src/channels/web/auth.rs b/src/channels/web/auth.rs index 9b1f5b47..b2fa4e4f 100644 --- a/src/channels/web/auth.rs +++ b/src/channels/web/auth.rs @@ -83,14 +83,15 @@ pub async fn auth_middleware( #[cfg(test)] mod tests { use super::*; + use crate::testing::credentials::{TEST_AUTH_SECRET_TOKEN, TEST_BEARER_TOKEN}; #[test] fn test_auth_state_clone() { let state = AuthState { - token: "test-token".to_string(), + token: TEST_BEARER_TOKEN.to_string(), }; let cloned = state.clone(); - assert_eq!(cloned.token, "test-token"); + assert_eq!(cloned.token, TEST_BEARER_TOKEN); } use axum::Router; @@ -120,10 +121,10 @@ mod tests { #[tokio::test] async fn test_valid_bearer_token_passes() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") - .header("Authorization", "Bearer secret-token") + .header("Authorization", format!("Bearer {TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -132,7 +133,7 @@ mod tests { #[tokio::test] async fn test_invalid_bearer_token_rejected() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") .header("Authorization", "Bearer wrong-token") @@ -144,9 +145,9 @@ mod tests { #[tokio::test] async fn test_query_token_allowed_for_chat_events() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() - .uri("/api/chat/events?token=secret-token") + .uri(format!("/api/chat/events?token={TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -155,9 +156,9 @@ mod tests { #[tokio::test] async fn test_query_token_allowed_for_logs_events() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() - .uri("/api/logs/events?token=secret-token") + .uri(format!("/api/logs/events?token={TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -166,9 +167,9 @@ mod tests { #[tokio::test] async fn test_query_token_allowed_for_ws_upgrade() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() - .uri("/api/chat/ws?token=secret-token") + .uri(format!("/api/chat/ws?token={TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -202,9 +203,9 @@ mod tests { #[tokio::test] async fn test_query_token_rejected_for_non_sse_get() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() - .uri("/api/chat/history?token=secret-token") + .uri(format!("/api/chat/history?token={TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -213,10 +214,10 @@ mod tests { #[tokio::test] async fn test_query_token_rejected_for_post() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .method(Method::POST) - .uri("/api/chat/send?token=secret-token") + .uri(format!("/api/chat/send?token={TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -225,7 +226,7 @@ mod tests { #[tokio::test] async fn test_query_token_invalid_rejected() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events?token=wrong-token") .body(Body::empty()) @@ -236,7 +237,7 @@ mod tests { #[tokio::test] async fn test_no_auth_at_all_rejected() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") .body(Body::empty()) @@ -247,11 +248,11 @@ mod tests { #[tokio::test] async fn test_bearer_header_works_for_post() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .method(Method::POST) .uri("/api/chat/send") - .header("Authorization", "Bearer secret-token") + .header("Authorization", format!("Bearer {TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -260,10 +261,10 @@ mod tests { #[tokio::test] async fn test_bearer_prefix_case_insensitive() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") - .header("Authorization", "bearer secret-token") + .header("Authorization", format!("bearer {TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -272,10 +273,10 @@ mod tests { #[tokio::test] async fn test_bearer_prefix_mixed_case() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") - .header("Authorization", "BEARER secret-token") + .header("Authorization", format!("BEARER {TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); @@ -284,7 +285,7 @@ mod tests { #[tokio::test] async fn test_empty_bearer_token_rejected() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") .header("Authorization", "Bearer ") @@ -296,10 +297,10 @@ mod tests { #[tokio::test] async fn test_token_with_whitespace_rejected() { - let app = test_app("secret-token"); + let app = test_app(TEST_AUTH_SECRET_TOKEN); let req = Request::builder() .uri("/api/chat/events") - .header("Authorization", "Bearer secret-token") + .header("Authorization", format!("Bearer {TEST_AUTH_SECRET_TOKEN}")) .body(Body::empty()) .unwrap(); let resp = app.oneshot(req).await.unwrap(); diff --git a/src/channels/web/handlers/chat.rs b/src/channels/web/handlers/chat.rs index e82c2583..b7f4425c 100644 --- a/src/channels/web/handlers/chat.rs +++ b/src/channels/web/handlers/chat.rs @@ -35,6 +35,7 @@ pub async fn chat_send_handler( } let msg_id = msg.id; + let thread_id = msg.thread_id.clone(); let tx_guard = state.msg_tx.read().await; let tx = tx_guard.as_ref().ok_or(( @@ -49,6 +50,13 @@ pub async fn chat_send_handler( ) })?; + tracing::debug!( + message_id = %msg_id, + thread_id = ?thread_id, + content_len = req.content.len(), + "Message queued to agent loop" + ); + Ok(( StatusCode::ACCEPTED, Json(SendMessageResponse { @@ -263,7 +271,6 @@ pub async fn chat_history_handler( ))?; let session = session_manager.get_or_create_session(&state.user_id).await; - let sess = session.lock().await; let limit = query.limit.unwrap_or(50); let before_cursor = query @@ -281,11 +288,12 @@ pub async fn chat_history_handler( }) .transpose()?; - // Find the thread + // Find the thread (lock only briefly to get active_thread if needed) let thread_id = if let Some(ref tid) = query.thread_id { Uuid::parse_str(tid) .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid thread_id".to_string()))? } else { + let sess = session.lock().await; sess.active_thread .ok_or((StatusCode::NOT_FOUND, "No active thread".to_string()))? }; @@ -298,8 +306,11 @@ pub async fn chat_history_handler( .conversation_belongs_to_user(thread_id, &state.user_id) .await .unwrap_or(false); - if !owned && !sess.threads.contains_key(&thread_id) { - return Err((StatusCode::NOT_FOUND, "Thread not found".to_string())); + if !owned { + let sess = session.lock().await; + if !sess.threads.contains_key(&thread_id) { + return Err((StatusCode::NOT_FOUND, "Thread not found".to_string())); + } } } @@ -324,56 +335,60 @@ pub async fn chat_history_handler( } // Try in-memory first (freshest data for active threads) - if let Some(thread) = sess.threads.get(&thread_id) - && (!thread.turns.is_empty() || thread.pending_approval.is_some()) + // Lock only when checking in-memory state { - let turns: Vec = thread - .turns - .iter() - .map(|t| TurnInfo { - turn_number: t.turn_number, - user_input: t.user_input.clone(), - response: t.response.clone(), - state: format!("{:?}", t.state), - started_at: t.started_at.to_rfc3339(), - completed_at: t.completed_at.map(|dt| dt.to_rfc3339()), - tool_calls: t - .tool_calls - .iter() - .map(|tc| ToolCallInfo { - name: tc.name.clone(), - has_result: tc.result.is_some(), - has_error: tc.error.is_some(), - result_preview: tc.result.as_ref().map(|r| { - let s = match r { - serde_json::Value::String(s) => s.clone(), - other => other.to_string(), - }; - truncate_preview(&s, 500) - }), - error: tc.error.clone(), - }) - .collect(), - }) - .collect(); + let sess = session.lock().await; + if let Some(thread) = sess.threads.get(&thread_id) + && (!thread.turns.is_empty() || thread.pending_approval.is_some()) + { + let turns: Vec = thread + .turns + .iter() + .map(|t| TurnInfo { + turn_number: t.turn_number, + user_input: t.user_input.clone(), + response: t.response.clone(), + state: format!("{:?}", t.state), + started_at: t.started_at.to_rfc3339(), + completed_at: t.completed_at.map(|dt| dt.to_rfc3339()), + tool_calls: t + .tool_calls + .iter() + .map(|tc| ToolCallInfo { + name: tc.name.clone(), + has_result: tc.result.is_some(), + has_error: tc.error.is_some(), + result_preview: tc.result.as_ref().map(|r| { + let s = match r { + serde_json::Value::String(s) => s.clone(), + other => other.to_string(), + }; + truncate_preview(&s, 500) + }), + error: tc.error.clone(), + }) + .collect(), + }) + .collect(); - let pending_approval = thread - .pending_approval - .as_ref() - .map(|pa| PendingApprovalInfo { - request_id: pa.request_id.to_string(), - tool_name: pa.tool_name.clone(), - description: pa.description.clone(), - parameters: serde_json::to_string_pretty(&pa.parameters).unwrap_or_default(), - }); + let pending_approval = thread + .pending_approval + .as_ref() + .map(|pa| PendingApprovalInfo { + request_id: pa.request_id.to_string(), + tool_name: pa.tool_name.clone(), + description: pa.description.clone(), + parameters: serde_json::to_string_pretty(&pa.parameters).unwrap_or_default(), + }); - return Ok(Json(HistoryResponse { - thread_id, - turns, - has_more: false, - oldest_timestamp: None, - pending_approval, - })); + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more: false, + oldest_timestamp: None, + pending_approval, + })); + } } // Fall back to DB for historical threads not in memory (paginated) @@ -415,7 +430,6 @@ pub async fn chat_threads_handler( ))?; let session = session_manager.get_or_create_session(&state.user_id).await; - let sess = session.lock().await; // Try DB first for persistent thread list if let Some(ref store) = state.store { @@ -465,15 +479,22 @@ pub async fn chat_threads_handler( }); } + // Read active thread while holding minimal lock (just before return) + let active_thread = { + let sess = session.lock().await; + sess.active_thread + }; + return Ok(Json(ThreadListResponse { assistant_thread, threads, - active_thread: sess.active_thread, + active_thread, })); } } // Fallback: in-memory only (no assistant thread without DB) + let sess = session.lock().await; let mut sorted_threads: Vec<_> = sess.threads.values().collect(); sorted_threads.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); let threads: Vec = sorted_threads @@ -490,10 +511,13 @@ pub async fn chat_threads_handler( }) .collect(); + let active_thread = sess.active_thread; + drop(sess); // Explicit drop to release lock + Ok(Json(ThreadListResponse { assistant_thread: None, threads, - active_thread: sess.active_thread, + active_thread, })) } diff --git a/src/channels/web/handlers/routines.rs b/src/channels/web/handlers/routines.rs index 8fbcc97b..d8803efa 100644 --- a/src/channels/web/handlers/routines.rs +++ b/src/channels/web/handlers/routines.rs @@ -27,7 +27,7 @@ pub async fn routines_list_handler( .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - let items: Vec = routines.iter().map(routine_to_info).collect(); + let items: Vec = routines.iter().map(RoutineInfo::from_routine).collect(); Ok(Json(RoutineListResponse { routines: items })) } @@ -263,54 +263,6 @@ pub async fn routines_runs_handler( }))) } -/// Convert a Routine to the trimmed RoutineInfo for list display. -fn routine_to_info(r: &crate::agent::routine::Routine) -> RoutineInfo { - let (trigger_type, trigger_summary) = match &r.trigger { - crate::agent::routine::Trigger::Cron { schedule, .. } => { - ("cron".to_string(), format!("cron: {}", schedule)) - } - crate::agent::routine::Trigger::Event { - pattern, channel, .. - } => { - let ch = channel.as_deref().unwrap_or("any"); - ("event".to_string(), format!("on {} /{}/", ch, pattern)) - } - crate::agent::routine::Trigger::Webhook { path, .. } => { - let p = path.as_deref().unwrap_or("/"); - ("webhook".to_string(), format!("webhook: {}", p)) - } - crate::agent::routine::Trigger::Manual => ("manual".to_string(), "manual only".to_string()), - }; - - let action_type = match &r.action { - crate::agent::routine::RoutineAction::Lightweight { .. } => "lightweight", - crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", - }; - - let status = if !r.enabled { - "disabled" - } else if r.consecutive_failures > 0 { - "failing" - } else { - "active" - }; - - RoutineInfo { - id: r.id, - name: r.name.clone(), - description: r.description.clone(), - enabled: r.enabled, - trigger_type, - trigger_summary, - action_type: action_type.to_string(), - last_run_at: r.last_run_at.map(|dt| dt.to_rfc3339()), - next_fire_at: r.next_fire_at.map(|dt| dt.to_rfc3339()), - run_count: r.run_count, - consecutive_failures: r.consecutive_failures, - status: status.to_string(), - } -} - /// Map `RoutineError` variants to appropriate HTTP status codes. fn routine_error_status(err: &RoutineError) -> StatusCode { match err { diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index 62c75c63..9c7561a1 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -663,9 +663,9 @@ async fn chat_send_handler( headers: axum::http::HeaderMap, Json(req): Json, ) -> Result<(StatusCode, Json), (StatusCode, String)> { - tracing::debug!( - "[chat_send_handler] Received message: content={:?}, thread_id={:?}", - req.content, + tracing::trace!( + "[chat_send_handler] Received message: content_len={}, thread_id={:?}", + req.content.len(), req.thread_id ); @@ -698,10 +698,10 @@ async fn chat_send_handler( } let msg_id = msg.id; - tracing::debug!( - "[chat_send_handler] Created message id={}, content={:?}, images={}", + tracing::trace!( + "[chat_send_handler] Created message id={}, content_len={}, images={}", msg_id, - req.content, + req.content.len(), req.images.len() ); @@ -1936,7 +1936,7 @@ async fn routines_list_handler( .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - let items: Vec = routines.iter().map(routine_to_info).collect(); + let items: Vec = routines.iter().map(RoutineInfo::from_routine).collect(); Ok(Json(RoutineListResponse { routines: items })) } @@ -2180,54 +2180,6 @@ async fn routines_runs_handler( }))) } -/// Convert a Routine to the trimmed RoutineInfo for list display. -fn routine_to_info(r: &crate::agent::routine::Routine) -> RoutineInfo { - let (trigger_type, trigger_summary) = match &r.trigger { - crate::agent::routine::Trigger::Cron { schedule, .. } => { - ("cron".to_string(), format!("cron: {}", schedule)) - } - crate::agent::routine::Trigger::Event { - pattern, channel, .. - } => { - let ch = channel.as_deref().unwrap_or("any"); - ("event".to_string(), format!("on {} /{}/", ch, pattern)) - } - crate::agent::routine::Trigger::Webhook { path, .. } => { - let p = path.as_deref().unwrap_or("/"); - ("webhook".to_string(), format!("webhook: {}", p)) - } - crate::agent::routine::Trigger::Manual => ("manual".to_string(), "manual only".to_string()), - }; - - let action_type = match &r.action { - crate::agent::routine::RoutineAction::Lightweight { .. } => "lightweight", - crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", - }; - - let status = if !r.enabled { - "disabled" - } else if r.consecutive_failures > 0 { - "failing" - } else { - "active" - }; - - RoutineInfo { - id: r.id, - name: r.name.clone(), - description: r.description.clone(), - enabled: r.enabled, - trigger_type, - trigger_summary, - action_type: action_type.to_string(), - last_run_at: r.last_run_at.map(|dt| dt.to_rfc3339()), - next_fire_at: r.next_fire_at.map(|dt| dt.to_rfc3339()), - run_count: r.run_count, - consecutive_failures: r.consecutive_failures, - status: status.to_string(), - } -} - // --- Settings handlers --- async fn settings_list_handler( @@ -2427,6 +2379,7 @@ struct GatewayStatusResponse { #[cfg(test)] mod tests { use super::*; + use crate::testing::credentials::TEST_GATEWAY_CRYPTO_KEY; #[test] fn test_build_turns_from_db_messages_complete() { @@ -2600,7 +2553,7 @@ mod tests { // Build an ExtensionManager so the handler can look up flows let secrets = Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "test-key-at-least-32-chars-long!!".to_string(), + TEST_GATEWAY_CRYPTO_KEY.to_string(), )) .expect("crypto"), ))); @@ -2650,7 +2603,7 @@ mod tests { let secrets: Arc = Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "test-key-at-least-32-chars-long!!".to_string(), + TEST_GATEWAY_CRYPTO_KEY.to_string(), )) .expect("crypto"), ))); @@ -2756,7 +2709,7 @@ mod tests { let secrets: Arc = Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "test-key-at-least-32-chars-long!!".to_string(), + TEST_GATEWAY_CRYPTO_KEY.to_string(), )) .expect("crypto"), ))); diff --git a/src/channels/web/static/style.css b/src/channels/web/static/style.css index 35c21702..2f1d9a53 100644 --- a/src/channels/web/static/style.css +++ b/src/channels/web/static/style.css @@ -1277,6 +1277,8 @@ body { gap: 8px; background: var(--bg-secondary); border-top: 1px solid var(--border); + flex-shrink: 0; + min-height: 56px; } .chat-input textarea { @@ -3720,6 +3722,21 @@ mark { .ext-install-form input { width: 100%; } + + /* Chat input: ensure visibility on mobile */ + .chat-input { + min-height: 52px; + } + + .chat-input textarea { + min-height: 36px; + max-height: 100px; + } + + .chat-input button { + padding: 6px 16px; + font-size: 14px; + } } /* Slash command autocomplete dropdown */ diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index b6d0d05a..b2355959 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -735,6 +735,60 @@ pub struct RoutineInfo { pub status: String, } +impl RoutineInfo { + /// Convert a `Routine` to the trimmed `RoutineInfo` for list display. + pub fn from_routine(r: &crate::agent::routine::Routine) -> Self { + let (trigger_type, trigger_summary) = match &r.trigger { + crate::agent::routine::Trigger::Cron { schedule, .. } => { + ("cron".to_string(), format!("cron: {}", schedule)) + } + crate::agent::routine::Trigger::Event { + pattern, channel, .. + } => { + let ch = channel.as_deref().unwrap_or("any"); + ("event".to_string(), format!("on {} /{}/", ch, pattern)) + } + crate::agent::routine::Trigger::SystemEvent { + source, event_type, .. + } => ( + "system_event".to_string(), + format!("event: {}.{}", source, event_type), + ), + crate::agent::routine::Trigger::Manual => { + ("manual".to_string(), "manual only".to_string()) + } + }; + + let action_type = match &r.action { + crate::agent::routine::RoutineAction::Lightweight { .. } => "lightweight", + crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", + }; + + let status = if !r.enabled { + "disabled" + } else if r.consecutive_failures > 0 { + "failing" + } else { + "active" + }; + + RoutineInfo { + id: r.id, + name: r.name.clone(), + description: r.description.clone(), + enabled: r.enabled, + trigger_type, + trigger_summary, + action_type: action_type.to_string(), + last_run_at: r.last_run_at.map(|dt| dt.to_rfc3339()), + next_fire_at: r.next_fire_at.map(|dt| dt.to_rfc3339()), + run_count: r.run_count, + consecutive_failures: r.consecutive_failures, + status: status.to_string(), + } + } +} + #[derive(Debug, Serialize)] pub struct RoutineListResponse { pub routines: Vec, diff --git a/src/channels/webhook_server.rs b/src/channels/webhook_server.rs index b56df912..f20d07e4 100644 --- a/src/channels/webhook_server.rs +++ b/src/channels/webhook_server.rs @@ -24,6 +24,8 @@ pub struct WebhookServerConfig { pub struct WebhookServer { config: WebhookServerConfig, routes: Vec, + /// Merged router saved after start() for restart_with_addr(). + merged_router: Option, shutdown_tx: Option>, handle: Option>, } @@ -34,6 +36,7 @@ impl WebhookServer { Self { config, routes: Vec::new(), + merged_router: None, shutdown_tx: None, handle: None, } @@ -51,7 +54,13 @@ impl WebhookServer { for fragment in self.routes.drain(..) { app = app.merge(fragment); } + self.merged_router = Some(app.clone()); + self.bind_and_spawn(app).await + } + /// Bind a listener to the configured address and spawn the server task. + /// Private helper used by both start() and restart_with_addr(). + async fn bind_and_spawn(&mut self, app: Router) -> Result<(), ChannelError> { let listener = tokio::net::TcpListener::bind(self.config.addr) .await .map_err(|e| ChannelError::StartupFailed { @@ -80,6 +89,54 @@ impl WebhookServer { Ok(()) } + /// Gracefully shut down the current listener and rebind to a new address. + /// The merged router from the original `start()` call is reused. + /// + /// If binding to the new address fails, the old listener remains active and + /// state is restored. This prevents a denial-of-service if the new address + /// is invalid or already in use. + pub async fn restart_with_addr(&mut self, new_addr: SocketAddr) -> Result<(), ChannelError> { + let app = self + .merged_router + .clone() + .ok_or_else(|| ChannelError::StartupFailed { + name: "webhook_server".to_string(), + reason: "restart_with_addr called before start()".to_string(), + })?; + + // Save old state for rollback if new bind fails + let old_addr = self.config.addr; + let old_shutdown_tx = self.shutdown_tx.take(); + let old_handle = self.handle.take(); + + // Update config to new address and try to bind + self.config.addr = new_addr; + match self.bind_and_spawn(app).await { + Ok(()) => { + // New listener is running, gracefully shut down the old one + if let Some(tx) = old_shutdown_tx { + let _ = tx.send(()); + } + if let Some(handle) = old_handle { + let _ = handle.await; + } + Ok(()) + } + Err(e) => { + // Restore old state; old listener remains active + self.config.addr = old_addr; + self.shutdown_tx = old_shutdown_tx; + self.handle = old_handle; + Err(e) + } + } + } + + /// Return the current bind address. + pub fn current_addr(&self) -> SocketAddr { + self.config.addr + } + /// Signal graceful shutdown and wait for the server task to finish. pub async fn shutdown(&mut self) { if let Some(tx) = self.shutdown_tx.take() { @@ -90,3 +147,182 @@ impl WebhookServer { } } } + +#[cfg(test)] +mod tests { + use super::*; + use axum::Json; + use serde_json::json; + + #[tokio::test] + async fn test_restart_with_addr_rebinds_listener() { + use std::net::TcpListener as StdTcpListener; + + // Find two available ports by binding and immediately closing + let port1 = { + let listener = + StdTcpListener::bind("127.0.0.1:0").expect("Failed to find available port 1"); + listener + .local_addr() + .expect("Failed to get local addr") + .port() + }; + + let port2 = { + let listener = + StdTcpListener::bind("127.0.0.1:0").expect("Failed to find available port 2"); + listener + .local_addr() + .expect("Failed to get local addr") + .port() + }; + + assert_ne!(port1, port2, "Should have different ports"); + assert_ne!(port1, 0, "Port 1 should be non-zero"); + assert_ne!(port2, 0, "Port 2 should be non-zero"); + + // Start server on first port + let addr1 = format!("127.0.0.1:{}", port1).parse().unwrap(); + let mut server = WebhookServer::new(WebhookServerConfig { addr: addr1 }); + + // Create a test router that responds to health checks + let test_router = axum::Router::new().route( + "/health", + axum::routing::get(|| async { Json(json!({"status": "ok"})) }), + ); + server.add_routes(test_router); + + // Start the server on first port + server.start().await.expect("Failed to start server"); + assert_eq!( + server.current_addr(), + addr1, + "Server should be bound to initial address" + ); + + // Verify the first server is actually listening + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{}/health", addr1)) + .send() + .await + .expect("Failed to send request to first server"); + assert_eq!( + response.status(), + 200, + "First server should respond to health check" + ); + + // Restart on second port + let addr2 = format!("127.0.0.1:{}", port2).parse().unwrap(); + server + .restart_with_addr(addr2) + .await + .expect("Failed to restart with new addr"); + + // Assert the address changed + assert_eq!( + server.current_addr(), + addr2, + "Server address should be updated after restart" + ); + assert_ne!( + addr1, addr2, + "Address should change after restart_with_addr" + ); + + // Verify the new server is actually listening on the new address + let response = client + .get(format!("http://{}/health", addr2)) + .send() + .await + .expect("Failed to send request to restarted server"); + assert_eq!( + response.status(), + 200, + "Restarted server should respond to health check on new address" + ); + + // Verify the old address is no longer responding + let old_result = tokio::time::timeout( + std::time::Duration::from_millis(200), + client.get(format!("http://{}/health", addr1)).send(), + ) + .await; + assert!( + old_result.is_err() || old_result.as_ref().unwrap().is_err(), + "Old address should not respond after server restarts" + ); + + // Clean up + server.shutdown().await; + } + + #[tokio::test] + async fn test_restart_with_addr_rollback_on_bind_failure() { + use std::net::TcpListener as StdTcpListener; + + // Find an available port + let port1 = { + let listener = + StdTcpListener::bind("127.0.0.1:0").expect("Failed to find available port"); + listener + .local_addr() + .expect("Failed to get local addr") + .port() + }; + + // Start server on first port + let addr1 = format!("127.0.0.1:{}", port1).parse().unwrap(); + let mut server = WebhookServer::new(WebhookServerConfig { addr: addr1 }); + + // Create a test router + let test_router = axum::Router::new().route( + "/health", + axum::routing::get(|| async { Json(json!({"status": "ok"})) }), + ); + server.add_routes(test_router); + + // Start the server on first port + server.start().await.expect("Failed to start server"); + + // Verify the server is listening + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{}/health", addr1)) + .send() + .await + .expect("Failed to send request"); + assert_eq!(response.status(), 200, "Server should be listening"); + + // Try to restart on an invalid address (port 0 is reserved, won't bind) + // Use port 1 which typically requires elevated privileges + let invalid_addr: SocketAddr = "127.0.0.1:1".parse().unwrap(); + + // Attempt restart (should fail) + let result = server.restart_with_addr(invalid_addr).await; + assert!(result.is_err(), "Restart with invalid address should fail"); + + // Verify the old address is still responding (rollback succeeded) + let response = client + .get(format!("http://{}/health", addr1)) + .send() + .await + .expect("Failed to send request to old address"); + assert_eq!( + response.status(), + 200, + "Old listener should still be running after failed restart" + ); + + // Verify the server address is unchanged + assert_eq!( + server.current_addr(), + addr1, + "Server address should be restored after failed restart" + ); + + // Clean up + server.shutdown().await; + } +} diff --git a/src/cli/doctor.rs b/src/cli/doctor.rs index c46f4863..aa47b6bf 100644 --- a/src/cli/doctor.rs +++ b/src/cli/doctor.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use crate::bootstrap::ironclaw_base_dir; +use crate::settings::Settings; /// Run all diagnostic checks and print results. pub async fn run_doctor_command() -> anyhow::Result<()> { @@ -15,14 +16,35 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { let mut passed = 0u32; let mut failed = 0u32; + let mut skipped = 0u32; - // ── Configuration checks ────────────────────────────────── + // Load settings once for checks that need them. + let settings = Settings::load(); + + // ── Settings & core config ───────────────────────────────── + + check( + "Settings file", + check_settings_file(), + &mut passed, + &mut failed, + &mut skipped, + ); check( "NEAR AI session", check_nearai_session().await, &mut passed, &mut failed, + &mut skipped, + ); + + check( + "LLM configuration", + check_llm_config(&settings), + &mut passed, + &mut failed, + &mut skipped, ); check( @@ -30,6 +52,7 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { check_database().await, &mut passed, &mut failed, + &mut skipped, ); check( @@ -37,15 +60,75 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { check_workspace_dir(), &mut passed, &mut failed, + &mut skipped, + ); + + // ── Subsystem configuration checks ───────────────────────── + + check( + "Embeddings", + check_embeddings(&settings), + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "Routines config", + check_routines_config(), + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "Gateway config", + check_gateway_config(&settings), + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "MCP servers", + check_mcp_config().await, + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "Skills", + check_skills().await, + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "Secrets", + check_secrets(&settings), + &mut passed, + &mut failed, + &mut skipped, + ); + + check( + "Service", + check_service_installed(), + &mut passed, + &mut failed, + &mut skipped, ); // ── External binary checks ──────────────────────────────── check( - "Docker", - check_binary("docker", &["--version"]), + "Docker daemon", + check_docker_daemon().await, &mut passed, &mut failed, + &mut skipped, ); check( @@ -53,6 +136,7 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { check_binary("cloudflared", &["--version"]), &mut passed, &mut failed, + &mut skipped, ); check( @@ -60,6 +144,7 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { check_binary("ngrok", &["version"]), &mut passed, &mut failed, + &mut skipped, ); check( @@ -67,12 +152,13 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { check_binary("tailscale", &["version"]), &mut passed, &mut failed, + &mut skipped, ); // ── Summary ─────────────────────────────────────────────── println!(); - println!(" {passed} passed, {failed} failed"); + println!(" {passed} passed, {failed} failed, {skipped} skipped"); if failed > 0 { println!("\n Some checks failed. This is normal if you don't use those features."); @@ -83,7 +169,7 @@ pub async fn run_doctor_command() -> anyhow::Result<()> { // ── Individual checks ─────────────────────────────────────── -fn check(name: &str, result: CheckResult, passed: &mut u32, failed: &mut u32) { +fn check(name: &str, result: CheckResult, passed: &mut u32, failed: &mut u32, skipped: &mut u32) { match result { CheckResult::Pass(detail) => { *passed += 1; @@ -94,6 +180,7 @@ fn check(name: &str, result: CheckResult, passed: &mut u32, failed: &mut u32) { println!(" [FAIL] {name}: {detail}"); } CheckResult::Skip(reason) => { + *skipped += 1; println!(" [skip] {name}: {reason}"); } } @@ -105,6 +192,29 @@ enum CheckResult { Skip(String), } +// ── Settings file ─────────────────────────────────────────── + +fn check_settings_file() -> CheckResult { + let path = Settings::default_path(); + if !path.exists() { + return CheckResult::Pass("no settings file (defaults will be used)".into()); + } + + match std::fs::read_to_string(&path) { + Ok(data) => match serde_json::from_str::(&data) { + Ok(_) => CheckResult::Pass(format!("valid ({})", path.display())), + Err(e) => CheckResult::Fail(format!( + "settings.json is malformed: {}. Fix or delete {}", + e, + path.display() + )), + }, + Err(e) => CheckResult::Fail(format!("cannot read {}: {}", path.display(), e)), + } +} + +// ── NEAR AI session ───────────────────────────────────────── + async fn check_nearai_session() -> CheckResult { // Check if session file exists let session_path = crate::config::llm::default_session_path(); @@ -129,6 +239,27 @@ async fn check_nearai_session() -> CheckResult { } } +// ── LLM configuration ────────────────────────────────────── + +fn check_llm_config(settings: &Settings) -> CheckResult { + match crate::llm::LlmConfig::resolve(settings) { + Ok(config) => { + // Show the model for the active backend, not always nearai.model. + let model = if let Some(ref bedrock) = config.bedrock { + &bedrock.model + } else if let Some(ref provider) = config.provider { + &provider.model + } else { + &config.nearai.model + }; + CheckResult::Pass(format!("backend={}, model={}", config.backend, model)) + } + Err(e) => CheckResult::Fail(format!("LLM config error: {e}")), + } +} + +// ── Database ──────────────────────────────────────────────── + async fn check_database() -> CheckResult { let backend = std::env::var("DATABASE_BACKEND") .ok() @@ -192,6 +323,8 @@ async fn try_pg_connect() -> Result<(), String> { Err("postgres feature not compiled in".into()) } +// ── Workspace directory ───────────────────────────────────── + fn check_workspace_dir() -> CheckResult { let dir = ironclaw_base_dir(); @@ -206,6 +339,222 @@ fn check_workspace_dir() -> CheckResult { } } +// ── Embeddings ────────────────────────────────────────────── + +fn check_embeddings(settings: &Settings) -> CheckResult { + match crate::config::EmbeddingsConfig::resolve(settings) { + Ok(config) => { + if !config.enabled { + return CheckResult::Skip("disabled (set EMBEDDING_ENABLED=true)".into()); + } + let has_creds = match config.provider.as_str() { + "openai" => config.openai_api_key().is_some(), + "nearai" => { + // NearAiEmbeddings uses SessionManager::get_token() which + // only returns session tokens, NOT NEARAI_API_KEY + // (src/workspace/embeddings.rs:309, src/llm/session.rs:132). + let session_path = crate::config::llm::default_session_path(); + session_path.exists() + && std::fs::read_to_string(&session_path) + .map(|s| !s.trim().is_empty()) + .unwrap_or(false) + } + "ollama" => true, // local, no creds needed + _ => config.openai_api_key().is_some(), + }; + if has_creds { + CheckResult::Pass(format!( + "provider={}, model={}", + config.provider, config.model + )) + } else { + let hint = match config.provider.as_str() { + "nearai" => "run `ironclaw onboard` to create a session", + _ => "set OPENAI_API_KEY", + }; + CheckResult::Fail(format!( + "provider={} but credentials missing ({})", + config.provider, hint + )) + } + } + Err(e) => CheckResult::Fail(format!("config error: {e}")), + } +} + +// ── Routines config ───────────────────────────────────────── + +fn check_routines_config() -> CheckResult { + match crate::config::RoutineConfig::resolve() { + Ok(config) => { + if config.enabled { + CheckResult::Pass(format!( + "enabled (interval={}s, max_concurrent={})", + config.cron_check_interval_secs, config.max_concurrent_routines + )) + } else { + CheckResult::Skip("disabled".into()) + } + } + Err(e) => CheckResult::Fail(format!("config error: {e}")), + } +} + +// ── Gateway config ────────────────────────────────────────── + +fn check_gateway_config(settings: &Settings) -> CheckResult { + // Use the same resolve() path as runtime so invalid env values + // (e.g. GATEWAY_PORT=abc) are caught here too. + match crate::config::ChannelsConfig::resolve(settings) { + Ok(channels) => match channels.gateway { + Some(gw) => { + if gw.auth_token.is_some() { + CheckResult::Pass(format!( + "enabled at {}:{} (auth token set)", + gw.host, gw.port + )) + } else { + CheckResult::Pass(format!( + "enabled at {}:{} (no auth token — random token will be generated)", + gw.host, gw.port + )) + } + } + None => CheckResult::Skip("disabled (GATEWAY_ENABLED=false)".into()), + }, + Err(e) => CheckResult::Fail(format!("config error: {e}")), + } +} + +// ── MCP servers ───────────────────────────────────────────── + +async fn check_mcp_config() -> CheckResult { + match crate::tools::mcp::config::load_mcp_servers().await { + Ok(file) => { + let servers: Vec<_> = file.enabled_servers().collect(); + if servers.is_empty() { + return CheckResult::Skip("no MCP servers configured".into()); + } + + let mut invalid = Vec::new(); + for server in &servers { + if let Err(e) = server.validate() { + invalid.push(format!("{}: {}", server.name, e)); + } + } + + if invalid.is_empty() { + CheckResult::Pass(format!("{} server(s) configured, all valid", servers.len())) + } else { + CheckResult::Fail(format!( + "{} server(s), {} invalid: {}", + servers.len(), + invalid.len(), + invalid.join("; ") + )) + } + } + Err(e) => { + // Distinguish no config from corrupted config + let msg = e.to_string(); + if msg.contains("not found") || msg.contains("No such file") { + CheckResult::Skip("no MCP config file".into()) + } else { + CheckResult::Fail(format!("config error: {e}")) + } + } + } +} + +// ── Skills ────────────────────────────────────────────────── + +async fn check_skills() -> CheckResult { + let user_dir = ironclaw_base_dir().join("skills"); + let installed_dir = ironclaw_base_dir().join("installed_skills"); + + let mut registry = crate::skills::SkillRegistry::new(user_dir.clone()); + registry = registry.with_installed_dir(installed_dir); + + // discover_all() returns loaded skill names (not warnings). + let _loaded_names = registry.discover_all().await; + + let count = registry.count(); + if count == 0 { + return CheckResult::Skip("no skills discovered".into()); + } + + CheckResult::Pass(format!("{count} skill(s) loaded")) +} + +// ── Secrets ───────────────────────────────────────────────── + +fn check_secrets(settings: &Settings) -> CheckResult { + match settings.secrets_master_key_source { + crate::settings::KeySource::Keychain => { + CheckResult::Pass("master key source: OS keychain".into()) + } + crate::settings::KeySource::Env => { + if std::env::var("SECRETS_MASTER_KEY").is_ok() { + CheckResult::Pass("master key source: env var (set)".into()) + } else { + CheckResult::Fail( + "master key source: env var but SECRETS_MASTER_KEY not set".into(), + ) + } + } + crate::settings::KeySource::None => { + CheckResult::Skip("secrets not configured (run `ironclaw onboard`)".into()) + } + } +} + +// ── Service ───────────────────────────────────────────────── + +fn check_service_installed() -> CheckResult { + if cfg!(target_os = "macos") { + let plist = + dirs::home_dir().map(|h| h.join("Library/LaunchAgents/com.ironclaw.daemon.plist")); + match plist { + Some(path) if path.exists() => { + CheckResult::Pass(format!("launchd plist installed ({})", path.display())) + } + Some(_) => CheckResult::Skip("not installed (run `ironclaw service install`)".into()), + None => CheckResult::Skip("cannot determine home directory".into()), + } + } else if cfg!(target_os = "linux") { + let unit = dirs::home_dir().map(|h| h.join(".config/systemd/user/ironclaw.service")); + match unit { + Some(path) if path.exists() => { + CheckResult::Pass(format!("systemd unit installed ({})", path.display())) + } + Some(_) => CheckResult::Skip("not installed (run `ironclaw service install`)".into()), + None => CheckResult::Skip("cannot determine home directory".into()), + } + } else { + CheckResult::Skip("service management not supported on this platform".into()) + } +} + +// ── Docker daemon ─────────────────────────────────────────── + +async fn check_docker_daemon() -> CheckResult { + let detection = crate::sandbox::check_docker().await; + match detection.status { + crate::sandbox::DockerStatus::Available => CheckResult::Pass("running".into()), + crate::sandbox::DockerStatus::NotInstalled => CheckResult::Skip(format!( + "not installed. {}", + detection.platform.install_hint() + )), + crate::sandbox::DockerStatus::NotRunning => CheckResult::Fail(format!( + "installed but not running. {}", + detection.platform.start_hint() + )), + crate::sandbox::DockerStatus::Disabled => CheckResult::Skip("sandbox disabled".into()), + } +} + +// ── External binary ───────────────────────────────────────── + fn check_binary(name: &str, args: &[&str]) -> CheckResult { match std::process::Command::new(name) .args(args) @@ -273,6 +622,193 @@ mod tests { } } + #[test] + fn check_settings_file_handles_missing() { + // Settings::default_path() might or might not exist, but must not panic + let result = check_settings_file(); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_llm_config_does_not_panic() { + let settings = Settings::default(); + let result = check_llm_config(&settings); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_routines_config_does_not_panic() { + let result = check_routines_config(); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_gateway_config_does_not_panic() { + let settings = Settings::default(); + let result = check_gateway_config(&settings); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_embeddings_does_not_panic() { + let settings = Settings::default(); + let result = check_embeddings(&settings); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_secrets_none_returns_skip() { + let settings = Settings::default(); + match check_secrets(&settings) { + CheckResult::Skip(msg) => { + assert!( + msg.contains("not configured"), + "expected 'not configured' in skip message, got: {msg}" + ); + } + other => panic!( + "expected Skip for default settings, got: {}", + format_result(&other) + ), + } + } + + #[test] + fn check_service_installed_does_not_panic() { + let result = check_service_installed(); + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[tokio::test] + async fn check_docker_daemon_does_not_panic() { + let result = check_docker_daemon().await; + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[tokio::test] + async fn check_mcp_config_does_not_panic() { + let result = check_mcp_config().await; + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[tokio::test] + async fn check_skills_does_not_panic() { + let result = check_skills().await; + match result { + CheckResult::Pass(_) | CheckResult::Fail(_) | CheckResult::Skip(_) => {} + } + } + + #[test] + fn check_llm_config_shows_nearai_model_for_nearai_backend() { + let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex"); + // SAFETY: Under ENV_MUTEX, no concurrent env access. + unsafe { + std::env::remove_var("LLM_BACKEND"); + } + let settings = Settings::default(); + match check_llm_config(&settings) { + CheckResult::Pass(msg) => { + assert!( + msg.contains("backend=nearai"), + "expected nearai backend, got: {msg}" + ); + // Must NOT show a bedrock or registry model when backend is nearai + assert!( + !msg.contains("anthropic.claude"), + "should not show bedrock model for nearai backend: {msg}" + ); + } + other => panic!( + "expected Pass for default LLM config, got: {}", + format_result(&other) + ), + } + } + + #[test] + fn check_embeddings_disabled_by_default_returns_skip() { + let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::remove_var("EMBEDDING_ENABLED"); + } + let settings = Settings::default(); + match check_embeddings(&settings) { + CheckResult::Skip(msg) => { + assert!( + msg.contains("disabled"), + "expected 'disabled' in skip message, got: {msg}" + ); + } + other => panic!( + "expected Skip for disabled embeddings, got: {}", + format_result(&other) + ), + } + } + + #[test] + fn check_routines_enabled_by_default() { + let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::remove_var("ROUTINES_ENABLED"); + } + match check_routines_config() { + CheckResult::Pass(msg) => { + assert!( + msg.contains("enabled"), + "routines should be enabled by default, got: {msg}" + ); + } + other => panic!( + "expected Pass for default routines, got: {}", + format_result(&other) + ), + } + } + + #[test] + fn check_secrets_env_without_var_returns_fail() { + let settings = Settings { + secrets_master_key_source: crate::settings::KeySource::Env, + ..Default::default() + }; + match check_secrets(&settings) { + CheckResult::Fail(msg) => { + assert!( + msg.contains("SECRETS_MASTER_KEY not set"), + "expected mention of missing env var, got: {msg}" + ); + } + CheckResult::Pass(_) => { + // If SECRETS_MASTER_KEY happens to be set in the environment, + // Pass is correct — don't fail the test. + } + other => panic!( + "expected Fail or Pass for env key source, got: {}", + format_result(&other) + ), + } + } + fn format_result(r: &CheckResult) -> String { match r { CheckResult::Pass(s) => format!("Pass({s})"), diff --git a/src/config/embeddings.rs b/src/config/embeddings.rs index 80719778..c5a84c00 100644 --- a/src/config/embeddings.rs +++ b/src/config/embeddings.rs @@ -154,6 +154,7 @@ mod tests { use super::*; use crate::config::helpers::ENV_MUTEX; use crate::settings::{EmbeddingsSettings, Settings}; + use crate::testing::credentials::*; /// Clear all embedding-related env vars. fn clear_embedding_env() { @@ -173,7 +174,7 @@ mod tests { clear_embedding_env(); // SAFETY: Under ENV_MUTEX, no concurrent env access. unsafe { - std::env::set_var("OPENAI_API_KEY", "sk-test-key-for-issue-129"); + std::env::set_var("OPENAI_API_KEY", TEST_OPENAI_API_KEY_ISSUE_129); } let settings = Settings { diff --git a/src/config/llm.rs b/src/config/llm.rs index cc02cd31..dd2c9563 100644 --- a/src/config/llm.rs +++ b/src/config/llm.rs @@ -389,6 +389,7 @@ mod tests { use super::*; use crate::config::helpers::ENV_MUTEX; use crate::settings::Settings; + use crate::testing::credentials::*; /// Clear all openai-compatible-related env vars. fn clear_openai_compatible_env() { @@ -657,7 +658,7 @@ mod tests { // SAFETY: Under ENV_MUTEX. unsafe { std::env::set_var("LLM_BACKEND", "open_ai"); - std::env::set_var("OPENAI_API_KEY", "test-key"); + std::env::set_var("OPENAI_API_KEY", TEST_API_KEY); } let settings = Settings::default(); @@ -791,7 +792,7 @@ mod tests { clear_anthropic_env(); // SAFETY: Under ENV_MUTEX. unsafe { - std::env::set_var("ANTHROPIC_OAUTH_TOKEN", "sk-ant-oat01-test-token"); + std::env::set_var("ANTHROPIC_OAUTH_TOKEN", TEST_ANTHROPIC_OAUTH_TOKEN); } let settings = Settings { @@ -815,7 +816,7 @@ mod tests { ); assert_eq!( provider.oauth_token.as_ref().unwrap().expose_secret(), - "sk-ant-oat01-test-token" + TEST_ANTHROPIC_OAUTH_TOKEN ); clear_anthropic_env(); @@ -829,8 +830,8 @@ mod tests { clear_anthropic_env(); // SAFETY: Under ENV_MUTEX. unsafe { - std::env::set_var("ANTHROPIC_API_KEY", "sk-ant-real-key"); - std::env::set_var("ANTHROPIC_OAUTH_TOKEN", "sk-ant-oat01-test-token"); + std::env::set_var("ANTHROPIC_API_KEY", TEST_ANTHROPIC_API_KEY); + std::env::set_var("ANTHROPIC_OAUTH_TOKEN", TEST_ANTHROPIC_OAUTH_TOKEN); } let settings = Settings { @@ -845,7 +846,7 @@ mod tests { .api_key .as_ref() .map(|k| k.expose_secret().to_string()), - Some("sk-ant-real-key".to_string()), + Some(TEST_ANTHROPIC_API_KEY.to_string()), "real API key should take priority over OAuth placeholder" ); assert!( @@ -862,7 +863,7 @@ mod tests { clear_anthropic_env(); // SAFETY: Under ENV_MUTEX. unsafe { - std::env::set_var("ANTHROPIC_OAUTH_TOKEN", "sk-ant-oat01-test-token"); + std::env::set_var("ANTHROPIC_OAUTH_TOKEN", TEST_ANTHROPIC_OAUTH_TOKEN); } let settings = Settings { diff --git a/src/config/sandbox.rs b/src/config/sandbox.rs index d757822d..35be4393 100644 --- a/src/config/sandbox.rs +++ b/src/config/sandbox.rs @@ -272,6 +272,7 @@ fn parse_oauth_access_token(json: &str) -> Option { #[cfg(test)] mod tests { use crate::config::sandbox::*; + use crate::testing::credentials::*; // ── SandboxModeConfig defaults ────────────────────────────────── @@ -405,9 +406,12 @@ mod tests { #[test] fn parse_oauth_token_valid() { - let json = r#"{"claudeAiOauth": {"accessToken": "sk-ant-oat01-fake"}}"#; - let token = parse_oauth_access_token(json); - assert_eq!(token, Some("sk-ant-oat01-fake".to_string())); + let json = format!( + r#"{{"claudeAiOauth": {{"accessToken": "{}"}}}}"#, + TEST_ANTHROPIC_OAUTH_BASIC + ); + let token = parse_oauth_access_token(&json); + assert_eq!(token, Some(TEST_ANTHROPIC_OAUTH_BASIC.to_string())); } #[test] @@ -434,16 +438,19 @@ mod tests { #[test] fn parse_oauth_token_nested_extra_fields() { - let json = r#"{ - "claudeAiOauth": { - "accessToken": "sk-ant-oat01-real-token", + let json = format!( + r#"{{ + "claudeAiOauth": {{ + "accessToken": "{}", "refreshToken": "rt-abc", "expiresAt": 1700000000 - } - }"#; + }} + }}"#, + TEST_ANTHROPIC_OAUTH_NESTED + ); assert_eq!( - parse_oauth_access_token(json), - Some("sk-ant-oat01-real-token".to_string()) + parse_oauth_access_token(&json), + Some(TEST_ANTHROPIC_OAUTH_NESTED.to_string()) ); } diff --git a/src/db/libsql/jobs.rs b/src/db/libsql/jobs.rs index 0750873d..3db3ab30 100644 --- a/src/db/libsql/jobs.rs +++ b/src/db/libsql/jobs.rs @@ -30,8 +30,9 @@ impl JobStore for LibSqlBackend { id, conversation_id, title, description, category, status, source, user_id, budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, - actual_cost, repair_attempts, created_at, started_at, completed_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18) + actual_cost, repair_attempts, max_tokens, total_tokens_used, + created_at, started_at, completed_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20) ON CONFLICT (id) DO UPDATE SET title = excluded.title, description = excluded.description, @@ -42,6 +43,8 @@ impl JobStore for LibSqlBackend { estimated_time_secs = excluded.estimated_time_secs, actual_cost = excluded.actual_cost, repair_attempts = excluded.repair_attempts, + max_tokens = excluded.max_tokens, + total_tokens_used = excluded.total_tokens_used, started_at = excluded.started_at, completed_at = excluded.completed_at "#, @@ -61,6 +64,8 @@ impl JobStore for LibSqlBackend { estimated_time_secs, ctx.actual_cost.to_string(), ctx.repair_attempts as i64, + ctx.max_tokens as i64, + ctx.total_tokens_used as i64, fmt_ts(&ctx.created_at), fmt_opt_ts(&ctx.started_at), fmt_opt_ts(&ctx.completed_at), @@ -78,7 +83,8 @@ impl JobStore for LibSqlBackend { r#" SELECT id, conversation_id, title, description, category, status, user_id, budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, - actual_cost, repair_attempts, created_at, started_at, completed_at + actual_cost, repair_attempts, max_tokens, total_tokens_used, + created_at, started_at, completed_at FROM agent_jobs WHERE id = ?1 "#, params![id.to_string()], @@ -111,12 +117,12 @@ impl JobStore for LibSqlBackend { estimated_duration: estimated_time_secs .map(|s| std::time::Duration::from_secs(s as u64)), actual_cost: get_decimal(&row, 12), - total_tokens_used: 0, - max_tokens: 0, + max_tokens: get_i64(&row, 14) as u64, + total_tokens_used: get_i64(&row, 15) as u64, repair_attempts: get_i64(&row, 13) as u32, - created_at: get_ts(&row, 14), - started_at: get_opt_ts(&row, 15), - completed_at: get_opt_ts(&row, 16), + created_at: get_ts(&row, 16), + started_at: get_opt_ts(&row, 17), + completed_at: get_opt_ts(&row, 18), transitions: Vec::new(), metadata: serde_json::Value::Null, extra_env: std::sync::Arc::new(std::collections::HashMap::new()), diff --git a/src/db/libsql/routines.rs b/src/db/libsql/routines.rs index f85ba0e3..3f2629ea 100644 --- a/src/db/libsql/routines.rs +++ b/src/db/libsql/routines.rs @@ -167,7 +167,7 @@ impl RoutineStore for LibSqlBackend { let mut rows = conn .query( &format!( - "SELECT {} FROM routines WHERE enabled = 1 AND trigger_type = 'event'", + "SELECT {} FROM routines WHERE enabled = 1 AND trigger_type IN ('event', 'system_event')", ROUTINE_COLUMNS ), (), diff --git a/src/db/libsql_migrations.rs b/src/db/libsql_migrations.rs index 63708235..fc445b7c 100644 --- a/src/db/libsql_migrations.rs +++ b/src/db/libsql_migrations.rs @@ -583,20 +583,21 @@ INSERT OR IGNORE INTO leak_detection_patterns (id, name, pattern, severity, acti /// /// Each entry is `(version, name, sql)`. Migrations are idempotent: the /// `_migrations` table tracks which versions have been applied. -pub const INCREMENTAL_MIGRATIONS: &[(i64, &str, &str)] = &[( - 9, - "flexible_embedding_dimension", - // Rebuild memory_chunks to remove the fixed F32_BLOB(1536) type - // constraint so any embedding dimension works. Existing embeddings - // are preserved; users only need to re-embed if they change models. - // - // The vector index (libsql_vector_idx) requires a fixed-dimension - // F32_BLOB(N), so we drop it entirely. Vector search falls back to - // brute-force cosine distance which is fast enough for personal - // assistant workspaces. This matches PostgreSQL after its V9 migration. - // - // SQLite cannot ALTER COLUMN types, so we recreate the table. - r#" +pub const INCREMENTAL_MIGRATIONS: &[(i64, &str, &str)] = &[ + ( + 9, + "flexible_embedding_dimension", + // Rebuild memory_chunks to remove the fixed F32_BLOB(1536) type + // constraint so any embedding dimension works. Existing embeddings + // are preserved; users only need to re-embed if they change models. + // + // The vector index (libsql_vector_idx) requires a fixed-dimension + // F32_BLOB(N), so we drop it entirely. Vector search falls back to + // brute-force cosine distance which is fast enough for personal + // assistant workspaces. This matches PostgreSQL after its V9 migration. + // + // SQLite cannot ALTER COLUMN types, so we recreate the table. + r#" -- Drop vector index (requires fixed F32_BLOB(N), incompatible with flexible dimensions) DROP INDEX IF EXISTS idx_memory_chunks_embedding; @@ -644,7 +645,18 @@ CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_update AFTER UPDATE ON memory_chu INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content); END; "#, -)]; + ), + ( + 12, + "job_token_budget", + // Add token budget tracking columns to agent_jobs. + // SQLite supports ALTER TABLE ADD COLUMN, so no table rebuild needed. + r#" +ALTER TABLE agent_jobs ADD COLUMN max_tokens INTEGER NOT NULL DEFAULT 0; +ALTER TABLE agent_jobs ADD COLUMN total_tokens_used INTEGER NOT NULL DEFAULT 0; +"#, + ), +]; /// Run incremental migrations that haven't been applied yet. /// @@ -653,6 +665,7 @@ END; pub async fn run_incremental(conn: &libsql::Connection) -> Result<(), crate::error::DatabaseError> { use crate::error::DatabaseError; + let mut applied_count = 0; for &(version, name, sql) in INCREMENTAL_MIGRATIONS { // Check if already applied let mut rows = conn @@ -669,8 +682,6 @@ pub async fn run_incremental(conn: &libsql::Connection) -> Result<(), crate::err continue; // Already applied } - tracing::info!(version, name, "libSQL: applying incremental migration"); - // Wrap migration + recording in a transaction for atomicity. // If the process crashes mid-migration, the transaction rolls back // and the migration will be retried on next startup. @@ -702,7 +713,12 @@ pub async fn run_incremental(conn: &libsql::Connection) -> Result<(), crate::err )) })?; - tracing::info!(version, name, "libSQL: migration applied successfully"); + applied_count += 1; + tracing::debug!(version, name, "libSQL: migration applied"); + } + + if applied_count > 0 { + tracing::info!("libSQL: applied {} incremental migrations", applied_count); } Ok(()) diff --git a/src/extensions/manager.rs b/src/extensions/manager.rs index 5e74c344..85d1ce74 100644 --- a/src/extensions/manager.rs +++ b/src/extensions/manager.rs @@ -113,6 +113,31 @@ pub struct ExtensionManager { gateway_token: Option, } +/// Sanitize a URL for logging by removing query parameters and credentials. +/// Prevents accidental logging of API keys, OAuth tokens, or other sensitive data in URLs. +fn sanitize_url_for_logging(url: &str) -> String { + // If URL is very short or doesn't look like a URL, just use as-is + if url.len() < 10 || !url.contains("://") { + return url.to_string(); + } + + // Try to parse and remove sensitive components + if let Ok(mut parsed) = url::Url::parse(url) { + // Remove query string and fragment + parsed.set_query(None); + parsed.set_fragment(None); + + // Remove userinfo (username and password) if present + let _ = parsed.set_username(""); + let _ = parsed.set_password(None); + + parsed.to_string() + } else { + // Fallback: strip after ? or # + url.split(['?', '#']).next().unwrap_or(url).to_string() + } +} + impl ExtensionManager { #[allow(clippy::too_many_arguments)] pub fn new( @@ -299,7 +324,8 @@ impl ExtensionManager { url: Option<&str>, kind_hint: Option, ) -> Result { - tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension"); + let sanitized_url = url.map(sanitize_url_for_logging); + tracing::info!(extension = %name, url = ?sanitized_url, kind = ?kind_hint, "Installing extension"); Self::validate_extension_name(name)?; // If we have a registry entry, use it (prefer kind_hint to resolve collisions) @@ -321,7 +347,8 @@ impl ExtensionManager { } } .map_err(|e| { - tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed"); + let sanitized = sanitize_url_for_logging(url); + tracing::error!(extension = %name, url = %sanitized, error = %e, "Extension install from URL failed"); e }); } @@ -1212,10 +1239,11 @@ impl ExtensionManager { .build() .map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?; - tracing::debug!(extension = %name, url = %url, "Downloading WASM extension"); + let sanitized_url = sanitize_url_for_logging(url); + tracing::debug!(extension = %name, url = %sanitized_url, "Downloading WASM extension"); let response = client.get(url).send().await.map_err(|e| { - tracing::error!(extension = %name, url = %url, error = %e, "Download request failed"); + tracing::error!(extension = %name, url = %sanitized_url, error = %e, "Download request failed"); ExtensionError::DownloadFailed(e.to_string()) })?; @@ -1223,7 +1251,7 @@ impl ExtensionManager { let status = response.status(); tracing::error!( extension = %name, - url = %url, + url = %sanitized_url, status = %status, "Download returned non-success HTTP status" ); @@ -2775,9 +2803,9 @@ impl ExtensionManager { } // Inject credentials - match crate::extensions::manager::inject_channel_credentials_from_secrets( + match inject_channel_credentials_from_secrets( &channel_arc, - self.secrets.as_ref(), + Some(self.secrets.as_ref()), &channel_name, &self.user_id, ) @@ -2862,7 +2890,7 @@ impl ExtensionManager { // Re-inject credentials from secrets store into the running channel let cred_count = match inject_channel_credentials_from_secrets( &existing_channel, - self.secrets.as_ref(), + Some(self.secrets.as_ref()), name, &self.user_id, ) @@ -3441,48 +3469,131 @@ impl ExtensionManager { /// Looks for secrets matching the pattern `{channel_name}_*` and injects them /// as credential placeholders (e.g., `telegram_bot_token` -> `{TELEGRAM_BOT_TOKEN}`). /// +/// Falls back to environment variables starting with the uppercase channel name +/// prefix (e.g., `TELEGRAM_` for channel `telegram`) for missing credentials. +/// /// Returns the number of credentials injected. async fn inject_channel_credentials_from_secrets( channel: &Arc, - secrets: &dyn SecretsStore, + secrets: Option<&dyn SecretsStore>, channel_name: &str, user_id: &str, ) -> Result { - let all_secrets = secrets - .list(user_id) - .await - .map_err(|e| format!("Failed to list secrets: {}", e))?; - - let prefix = format!("{}_", channel_name); let mut count = 0; + let mut injected_placeholders = std::collections::HashSet::new(); - for secret_meta in all_secrets { - if !secret_meta.name.starts_with(&prefix) { - continue; - } + // 1. Try injecting from persistent secrets store if available + if let Some(secrets) = secrets { + let all_secrets = secrets + .list(user_id) + .await + .map_err(|e| format!("Failed to list secrets: {}", e))?; - let decrypted = match secrets.get_decrypted(user_id, &secret_meta.name).await { - Ok(d) => d, - Err(e) => { - tracing::warn!( - secret = %secret_meta.name, - error = %e, - "Failed to decrypt secret for channel credential injection" - ); + let prefix = format!("{}_", channel_name.to_ascii_lowercase()); + + for secret_meta in all_secrets { + if !secret_meta.name.to_ascii_lowercase().starts_with(&prefix) { continue; } - }; - let placeholder = secret_meta.name.to_uppercase(); - channel - .set_credential(&placeholder, decrypted.expose().to_string()) - .await; - count += 1; + let decrypted = match secrets.get_decrypted(user_id, &secret_meta.name).await { + Ok(d) => d, + Err(e) => { + tracing::warn!( + secret = %secret_meta.name, + error = %e, + "Failed to decrypt secret for channel credential injection" + ); + continue; + } + }; + + let placeholder = secret_meta.name.to_uppercase(); + channel + .set_credential(&placeholder, decrypted.expose().to_string()) + .await; + injected_placeholders.insert(placeholder); + count += 1; + } } + // 2. Fallback to environment variables for missing credentials + count += inject_env_credentials(channel, channel_name, &injected_placeholders).await; + Ok(count) } +/// Inject missing credentials from environment variables. +/// +/// Only environment variables starting with the uppercase channel name prefix +/// (e.g., `TELEGRAM_` for channel `telegram`) are considered for security. +async fn inject_env_credentials( + channel: &Arc, + channel_name: &str, + already_injected: &std::collections::HashSet, +) -> usize { + if channel_name.trim().is_empty() { + return 0; + } + + let caps = channel.capabilities(); + let Some(ref http_cap) = caps.tool_capabilities.http else { + return 0; + }; + + let placeholders: Vec = http_cap + .credentials + .values() + .map(|m| m.secret_name.to_uppercase()) + .collect(); + + let resolved = resolve_env_credentials(&placeholders, channel_name, already_injected); + let count = resolved.len(); + for (placeholder, value) in resolved { + channel.set_credential(&placeholder, value).await; + } + count +} + +/// Pure helper: from a list of credential placeholder names, return those that +/// pass the channel-prefix security check and have a non-empty env var value. +/// +/// Placeholders already covered by the secrets store (`already_injected`) are +/// skipped. Only names starting with `{CHANNEL_NAME}_` are allowed to prevent +/// a WASM channel from reading unrelated host credentials (e.g. `AWS_SECRET_ACCESS_KEY`). +pub(crate) fn resolve_env_credentials( + placeholders: &[String], + channel_name: &str, + already_injected: &std::collections::HashSet, +) -> Vec<(String, String)> { + if channel_name.trim().is_empty() { + return Vec::new(); + } + + let prefix = format!("{}_", channel_name.to_ascii_uppercase()); + let mut out = Vec::new(); + + for placeholder in placeholders { + if already_injected.contains(placeholder) { + continue; + } + if !placeholder.starts_with(&prefix) { + tracing::warn!( + channel = %channel_name, + placeholder = %placeholder, + "Ignoring non-prefixed credential placeholder in environment fallback" + ); + continue; + } + if let Ok(value) = std::env::var(placeholder) + && !value.is_empty() + { + out.push((placeholder.clone(), value)); + } + } + out +} + /// Infer the extension kind from a URL. fn infer_kind_from_url(url: &str) -> ExtensionKind { if url.ends_with(".wasm") || url.ends_with(".tar.gz") { @@ -3907,6 +4018,7 @@ mod tests { channels_dir: std::path::PathBuf, ) -> ExtensionManager { use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + use crate::testing::credentials::TEST_CRYPTO_KEY; use crate::tools::ToolRegistry; use crate::tools::mcp::process::McpProcessManager; use crate::tools::mcp::session::McpSessionManager; @@ -3914,8 +4026,7 @@ mod tests { std::fs::create_dir_all(&tools_dir).ok(); std::fs::create_dir_all(&channels_dir).ok(); - let master_key = - secrecy::SecretString::from("0123456789abcdef0123456789abcdef".to_string()); + let master_key = secrecy::SecretString::from(TEST_CRYPTO_KEY.to_string()); let crypto = Arc::new(SecretsCrypto::new(master_key).unwrap()); ExtensionManager::new( @@ -3933,4 +4044,187 @@ mod tests { Vec::new(), ) } + + // ── resolve_env_credentials tests ──────────────────────────────────── + + #[test] + fn test_security_prefix_check() { + // Placeholders that don't start with the channel prefix must be rejected. + // All env var names are prefixed with ICTEST1_ to avoid CI collisions. + let placeholders = vec![ + "ICTEST1_BOT_TOKEN".to_string(), // valid: matches channel prefix + "ICTEST2_TOKEN".to_string(), // invalid: wrong channel prefix + "ICTEST1_UNRELATED_OTHER".to_string(), // valid prefix, but env var not set — not injected + ]; + let already_injected = std::collections::HashSet::new(); + + unsafe { std::env::set_var("ICTEST1_BOT_TOKEN", "good-secret") }; + unsafe { std::env::set_var("ICTEST2_TOKEN", "bad-secret") }; + // ICTEST1_UNRELATED_OTHER intentionally not set — tests both prefix rejection and absence + + let resolved = super::resolve_env_credentials(&placeholders, "ictest1", &already_injected); + + // Only ICTEST1_BOT_TOKEN passes the prefix check for channel "ictest1" + assert_eq!(resolved.len(), 1); + assert_eq!(resolved[0].0, "ICTEST1_BOT_TOKEN"); + assert_eq!(resolved[0].1, "good-secret"); + + unsafe { std::env::remove_var("ICTEST1_BOT_TOKEN") }; + unsafe { std::env::remove_var("ICTEST2_TOKEN") }; + } + + #[test] + fn test_already_injected_skipped() { + // Use unique env var names (ictest3_*) to avoid interference with other tests. + let placeholders = vec!["ICTEST3_TOKEN".to_string()]; + let mut already_injected = std::collections::HashSet::new(); + already_injected.insert("ICTEST3_TOKEN".to_string()); + + unsafe { std::env::set_var("ICTEST3_TOKEN", "secret") }; + + let resolved = super::resolve_env_credentials(&placeholders, "ictest3", &already_injected); + + // Already covered by secrets store — env var must be skipped + assert!(resolved.is_empty()); + + unsafe { std::env::remove_var("ICTEST3_TOKEN") }; + } + + #[test] + fn test_missing_env_var_not_injected() { + // Use unique env var names (ictest4_*) to avoid interference with other tests. + let placeholders = vec!["ICTEST4_TOKEN".to_string()]; + let already_injected = std::collections::HashSet::new(); + + unsafe { std::env::remove_var("ICTEST4_TOKEN") }; + + let resolved = super::resolve_env_credentials(&placeholders, "ictest4", &already_injected); + + assert!(resolved.is_empty()); + } + + #[test] + fn test_empty_env_var_not_injected() { + // An env var that exists but is empty must not be injected. + // Use unique env var names (ictest5_*) to avoid interference with other tests. + let placeholders = vec!["ICTEST5_TOKEN".to_string()]; + let already_injected = std::collections::HashSet::new(); + + unsafe { std::env::set_var("ICTEST5_TOKEN", "") }; + + let resolved = super::resolve_env_credentials(&placeholders, "ictest5", &already_injected); + + assert!(resolved.is_empty()); + + unsafe { std::env::remove_var("ICTEST5_TOKEN") }; + } + + #[test] + fn test_empty_channel_name_returns_nothing() { + // An empty channel name must never match any env var (prefix would be "_"). + let placeholders = vec!["_TOKEN".to_string(), "ICTEST6_TOKEN".to_string()]; + let already_injected = std::collections::HashSet::new(); + + unsafe { std::env::set_var("_TOKEN", "bad") }; + unsafe { std::env::set_var("ICTEST6_TOKEN", "bad") }; + + let resolved = super::resolve_env_credentials(&placeholders, "", &already_injected); + + assert!(resolved.is_empty(), "empty channel name must match nothing"); + + unsafe { std::env::remove_var("_TOKEN") }; + unsafe { std::env::remove_var("ICTEST6_TOKEN") }; + } + + #[test] + fn test_sanitize_url_with_query_params() { + let url = "https://api.example.com/path?api_key=secret123&token=abc"; + let result = super::sanitize_url_for_logging(url); + assert_eq!(result, "https://api.example.com/path"); + assert!(!result.contains("api_key")); + assert!(!result.contains("secret123")); + assert!(!result.contains("token")); + } + + #[test] + fn test_sanitize_url_with_credentials() { + let url = "https://user:password@api.example.com:8080/path"; + let result = super::sanitize_url_for_logging(url); + assert!(!result.contains("user")); + assert!(!result.contains("password")); + assert!(!result.contains("@")); + assert!(result.contains("api.example.com")); + assert!(result.contains(":8080")); + } + + #[test] + fn test_sanitize_url_with_fragment() { + let url = "https://api.example.com/path#section"; + let result = super::sanitize_url_for_logging(url); + assert_eq!(result, "https://api.example.com/path"); + assert!(!result.contains("#")); + assert!(!result.contains("section")); + } + + #[test] + fn test_sanitize_url_with_port() { + let url = "https://api.example.com:9443/path?key=value"; + let result = super::sanitize_url_for_logging(url); + assert_eq!(result, "https://api.example.com:9443/path"); + assert!(result.contains(":9443")); + assert!(!result.contains("key")); + } + + #[test] + fn test_sanitize_url_with_all_components() { + let url = "https://admin:secret@api.example.com:8080/v1/data?api_key=xyz#results"; + let result = super::sanitize_url_for_logging(url); + assert!(!result.contains("admin")); + assert!(!result.contains("secret")); + assert!(!result.contains("@")); + assert!(!result.contains("api_key")); + assert!(!result.contains("xyz")); + assert!(!result.contains("#")); + assert!(!result.contains("results")); + assert!(result.contains("api.example.com:8080")); + assert!(result.contains("/v1/data")); + } + + #[test] + fn test_sanitize_url_malformed() { + // Malformed URL should fallback to string splitting + let url = "https://[invalid-url"; + let result = super::sanitize_url_for_logging(url); + // Malformed URL without query should return as-is via fallback + assert_eq!(result, url); + + // Should still strip query params via fallback + let url_with_query = "https://[invalid-url?key=secret"; + let result_with_query = super::sanitize_url_for_logging(url_with_query); + assert_eq!(result_with_query, "https://[invalid-url"); + assert!(!result_with_query.contains("?")); + assert!(!result_with_query.contains("secret")); + } + + #[test] + fn test_sanitize_url_short_string() { + let url = "short"; + let result = super::sanitize_url_for_logging(url); + assert_eq!(result, "short"); + } + + #[test] + fn test_sanitize_url_not_url_like() { + let input = "this is not a url"; + let result = super::sanitize_url_for_logging(input); + assert_eq!(result, input); + } + + #[test] + fn test_sanitize_url_preserves_path() { + let url = "https://api.example.com/v1/users/123/profile"; + let result = super::sanitize_url_for_logging(url); + assert_eq!(result, url); + assert!(result.contains("/v1/users/123/profile")); + } } diff --git a/src/history/store.rs b/src/history/store.rs index 1153f3e4..e877cbbf 100644 --- a/src/history/store.rs +++ b/src/history/store.rs @@ -151,8 +151,9 @@ impl Store { id, conversation_id, title, description, category, status, source, user_id, budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, - actual_cost, repair_attempts, created_at, started_at, completed_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) + actual_cost, repair_attempts, max_tokens, total_tokens_used, + created_at, started_at, completed_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20) ON CONFLICT (id) DO UPDATE SET title = EXCLUDED.title, description = EXCLUDED.description, @@ -163,6 +164,8 @@ impl Store { estimated_time_secs = EXCLUDED.estimated_time_secs, actual_cost = EXCLUDED.actual_cost, repair_attempts = EXCLUDED.repair_attempts, + max_tokens = EXCLUDED.max_tokens, + total_tokens_used = EXCLUDED.total_tokens_used, started_at = EXCLUDED.started_at, completed_at = EXCLUDED.completed_at "#, @@ -182,6 +185,8 @@ impl Store { &estimated_time_secs, &ctx.actual_cost, &(ctx.repair_attempts as i32), + &(ctx.max_tokens as i64), + &(ctx.total_tokens_used as i64), &ctx.created_at, &ctx.started_at, &ctx.completed_at, @@ -201,7 +206,8 @@ impl Store { r#" SELECT id, conversation_id, title, description, category, status, user_id, budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, - actual_cost, repair_attempts, created_at, started_at, completed_at + actual_cost, repair_attempts, max_tokens, total_tokens_used, + created_at, started_at, completed_at FROM agent_jobs WHERE id = $1 "#, &[&id], @@ -237,8 +243,9 @@ impl Store { completed_at: row.get("completed_at"), transitions: Vec::new(), // Not loaded from DB for now metadata: serde_json::Value::Null, - total_tokens_used: 0, - max_tokens: 0, + max_tokens: row.get::<_, Option>("max_tokens").unwrap_or(0) as u64, + total_tokens_used: row.get::<_, Option>("total_tokens_used").unwrap_or(0) + as u64, extra_env: std::sync::Arc::new(std::collections::HashMap::new()), http_interceptor: None, tool_output_stash: std::sync::Arc::new(tokio::sync::RwLock::new( @@ -1087,7 +1094,7 @@ impl Store { let conn = self.conn().await?; let rows = conn .query( - "SELECT * FROM routines WHERE enabled AND trigger_type = 'event'", + "SELECT * FROM routines WHERE enabled AND trigger_type IN ('event', 'system_event')", &[], ) .await?; diff --git a/src/llm/anthropic_oauth.rs b/src/llm/anthropic_oauth.rs index 0badda93..12ca223c 100644 --- a/src/llm/anthropic_oauth.rs +++ b/src/llm/anthropic_oauth.rs @@ -19,7 +19,8 @@ use crate::llm::costs; use crate::llm::error::LlmError; use crate::llm::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall, - ToolCompletionRequest, ToolCompletionResponse, + ToolCompletionRequest, ToolCompletionResponse, strip_unsupported_completion_params, + strip_unsupported_tool_params, }; const ANTHROPIC_API_URL: &str = "https://api.anthropic.com/v1/messages"; @@ -80,28 +81,12 @@ impl AnthropicOAuthProvider { /// Strip unsupported fields from a `CompletionRequest` in place. fn strip_unsupported_completion_params(&self, req: &mut CompletionRequest) { - if self.unsupported_params.is_empty() { - return; - } - if self.unsupported_params.contains("temperature") { - req.temperature = None; - } - if self.unsupported_params.contains("max_tokens") { - req.max_tokens = None; - } + strip_unsupported_completion_params(&self.unsupported_params, req); } /// Strip unsupported fields from a `ToolCompletionRequest` in place. fn strip_unsupported_tool_params(&self, req: &mut ToolCompletionRequest) { - if self.unsupported_params.is_empty() { - return; - } - if self.unsupported_params.contains("temperature") { - req.temperature = None; - } - if self.unsupported_params.contains("max_tokens") { - req.max_tokens = None; - } + strip_unsupported_tool_params(&self.unsupported_params, req); } fn api_url(&self) -> String { diff --git a/src/llm/mod.rs b/src/llm/mod.rs index c992f89c..b49e4974 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -29,6 +29,7 @@ pub mod session; pub mod smart_routing; pub mod image_models; +pub mod reasoning_models; pub mod vision_models; pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider}; diff --git a/src/llm/nearai_chat.rs b/src/llm/nearai_chat.rs index 3f4b4339..da99c080 100644 --- a/src/llm/nearai_chat.rs +++ b/src/llm/nearai_chat.rs @@ -270,8 +270,15 @@ impl NearAiChatProvider { reason: format!("Failed to read response body: {}", e), })?; - tracing::debug!("NEAR AI Chat response status: {}", status); - tracing::debug!("NEAR AI Chat response body: {}", response_text); + if tracing::enabled!(tracing::Level::DEBUG) { + tracing::debug!("NEAR AI Chat response status: {}", status); + } + + // Log response body only at TRACE level to avoid exposing sensitive content + // (user-generated data, tool outputs, leaked secrets) in DEBUG logs + if tracing::enabled!(tracing::Level::TRACE) { + tracing::trace!("NEAR AI Chat response body: {}", response_text); + } if !status.is_success() { let status_code = status.as_u16(); diff --git a/src/llm/provider.rs b/src/llm/provider.rs index 40ab8100..787bbff1 100644 --- a/src/llm/provider.rs +++ b/src/llm/provider.rs @@ -455,6 +455,73 @@ pub fn sanitize_tool_messages(messages: &mut [ChatMessage]) { } } +/// Represents a request parameter that may not be supported by all LLM providers. +/// +/// This typed enum replaces stringly-typed parameter names across the codebase, +/// providing type safety and single-point-of-maintenance for parameter handling. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum UnsupportedParam { + Temperature, + MaxTokens, + StopSequences, +} + +impl UnsupportedParam { + /// Get the string name of this parameter for config/error messages. + pub fn name(&self) -> &'static str { + match self { + UnsupportedParam::Temperature => "temperature", + UnsupportedParam::MaxTokens => "max_tokens", + UnsupportedParam::StopSequences => "stop_sequences", + } + } +} + +/// Strip unsupported parameters from a `CompletionRequest` in place. +/// +/// This is the single helper function used by all providers to remove +/// parameters they don't support, replacing duplicate stringly-typed logic. +pub fn strip_unsupported_completion_params( + unsupported: &std::collections::HashSet, + req: &mut CompletionRequest, +) { + if unsupported.is_empty() { + return; + } + if unsupported.contains(UnsupportedParam::Temperature.name()) { + req.temperature = None; + } + if unsupported.contains(UnsupportedParam::MaxTokens.name()) { + req.max_tokens = None; + } + if unsupported.contains(UnsupportedParam::StopSequences.name()) { + req.stop_sequences = None; + } +} + +/// Strip unsupported parameters from a `ToolCompletionRequest` in place. +/// +/// This is the single helper function used by all providers to remove +/// parameters they don't support from tool calls, replacing duplicate stringly-typed logic. +/// +/// Note: Only `Temperature` and `MaxTokens` are supported in `ToolCompletionRequest`. +/// `StopSequences` is only available in `CompletionRequest` and is not applicable to tool calls. +pub fn strip_unsupported_tool_params( + unsupported: &std::collections::HashSet, + req: &mut ToolCompletionRequest, +) { + if unsupported.is_empty() { + return; + } + if unsupported.contains(UnsupportedParam::Temperature.name()) { + req.temperature = None; + } + if unsupported.contains(UnsupportedParam::MaxTokens.name()) { + req.max_tokens = None; + } + // Note: StopSequences is not a field in ToolCompletionRequest, so no action needed +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/llm/reasoning.rs b/src/llm/reasoning.rs index 4b20865a..3a654fed 100644 --- a/src/llm/reasoning.rs +++ b/src/llm/reasoning.rs @@ -450,7 +450,8 @@ impl Reasoning { cache_read_input_tokens: response.cache_read_input_tokens, cache_creation_input_tokens: response.cache_creation_input_tokens, }; - Ok((clean_response(&response.content), usage)) + let pre_truncated = truncate_at_tool_tags(&response.content); + Ok((clean_response(&pre_truncated), usage)) } /// Generate a plan for completing a goal. @@ -480,8 +481,11 @@ impl Reasoning { let response = self.llm.complete(request).await?; - // Clean reasoning model artifacts before parsing JSON - let cleaned = clean_response(&response.content); + // Clean reasoning model artifacts before parsing JSON. + // Pre-truncate at tool tags to avoid strip_xml_tag discarding + // content after unclosed tags (issue #789). + let pre_truncated = truncate_at_tool_tags(&response.content); + let cleaned = clean_response(&pre_truncated); self.parse_plan(&cleaned) } @@ -575,8 +579,11 @@ Respond in JSON format: let response = self.llm.complete(request).await?; - // Clean reasoning model artifacts before parsing JSON - let cleaned = clean_response(&response.content); + // Clean reasoning model artifacts before parsing JSON. + // Pre-truncate at tool tags to avoid strip_xml_tag discarding + // content after unclosed tags (issue #789). + let pre_truncated = truncate_at_tool_tags(&response.content); + let cleaned = clean_response(&pre_truncated); self.parse_evaluation(&cleaned) } @@ -653,7 +660,10 @@ Respond in JSON format: return Ok(RespondOutput { result: RespondResult::ToolCalls { tool_calls: response.tool_calls, - content: response.content.map(|c| clean_response(&c)), + content: response.content.map(|c| { + let pre_truncated = truncate_at_tool_tags(&c); + clean_response(&pre_truncated) + }), }, usage, }); @@ -666,9 +676,13 @@ Respond in JSON format: // Some models (e.g. GLM-4.7) emit tool calls as XML tags in content // instead of using the structured tool_calls field. Try to recover // them before giving up and returning plain text. + // NOTE: Recovery runs on the raw content (before truncation) so it can + // parse tool-call JSON from the XML tags. Truncation only applies to the + // remaining *text* content returned alongside the recovered tool calls. let recovered = recover_tool_calls_from_content(&content, &context.available_tools); if !recovered.is_empty() { - let cleaned = clean_response(&content); + let pre_truncated = truncate_at_tool_tags(&content); + let cleaned = clean_response(&pre_truncated); return Ok(RespondOutput { result: RespondResult::ToolCalls { tool_calls: recovered, @@ -682,12 +696,16 @@ Respond in JSON format: }); } - // Guard against empty text after cleaning. This can happen - // when reasoning models (e.g. GLM-5) return chain-of-thought - // in reasoning_content wrapped in tags and content is - // null — the .or(reasoning_content) fallback picks it up, then - // clean_response strips the think tags leaving an empty string. - let cleaned = clean_response(&content); + // Guard against empty text after cleaning. This can happen when: + // 1. Reasoning models (e.g. GLM-5) return chain-of-thought in + // reasoning_content wrapped in tags — clean_response + // strips the think tags leaving an empty string. + // 2. Local models (Qwen3, DeepSeek) emit XML in text + // responses even in force_text mode — strip_xml_tag discards + // from unclosed opening tag onward (issue #789). + // Pre-truncate at tool tags to preserve text before the tag. + let pre_truncated = truncate_at_tool_tags(&content); + let cleaned = clean_response(&pre_truncated); let final_text = if cleaned.trim().is_empty() { tracing::warn!( "LLM response was empty after cleaning (original len={}), using fallback", @@ -709,7 +727,8 @@ Respond in JSON format: request.metadata = context.metadata.clone(); let response = self.llm.complete(request).await?; - let cleaned = clean_response(&response.content); + let pre_truncated = truncate_at_tool_tags(&response.content); + let cleaned = clean_response(&pre_truncated); let final_text = if cleaned.trim().is_empty() { tracing::warn!( "LLM response was empty after cleaning (original len={}), using fallback", @@ -847,10 +866,22 @@ Respond with a JSON plan in this format: .to_string() }; - format!( - r#"You are IronClaw Agent, a secure autonomous assistant. + // Models with native thinking (Qwen3, DeepSeek-R1, etc.) produce their + // own tags or reasoning_content. Injecting our / + // format collides with their native behavior, causing thinking-only + // responses that clean to empty strings. See issue #789. + let has_native_thinking = self + .model_name + .as_ref() + .is_some_and(|n| crate::llm::reasoning_models::has_native_thinking(n)); -## Response Format — CRITICAL + let response_format = if has_native_thinking { + r#"## Response Format + +Respond directly with your answer. Do not wrap your response in any special tags. +Your reasoning process is handled natively — just provide the final user-facing answer."# + } else { + r#"## Response Format — CRITICAL ALL internal reasoning MUST be inside ... tags. Do not output any analysis, planning, or self-talk outside . @@ -860,7 +891,13 @@ Only text inside is shown to the user; everything else is discarded. Example: The user is asking about X. -Here is the answer about X. +Here is the answer about X."# + }; + + format!( + r#"You are IronClaw Agent, a secure autonomous assistant. + +{response_format} ## Guidelines - Be concise and direct @@ -1442,6 +1479,99 @@ fn strip_bracket_tool_calls(text: &str) -> String { /// Tool-related tags stripped with simple string matching (no code-awareness needed). const TOOL_TAGS: &[&str] = &["tool_call", "function_call", "tool_calls"]; +/// Patterns that indicate tool-call XML in model output. +const TOOL_TAG_PATTERNS: &[&str] = &[ + "", + "", + "", + "", + "<|function_call|>", + "<|tool_calls|>", +]; + +/// Truncate text at the first **unclosed** tool-call XML tag, preserving content +/// before it. +/// +/// Local models (Qwen3, DeepSeek, etc.) often emit `` XML in text +/// responses even when no tools are available. The downstream `clean_response()` +/// → `strip_xml_tag()` pipeline discards everything from an unclosed opening +/// tag onward, which can leave an empty string and trigger the fallback message. +/// +/// This function truncates at the first *unclosed* tool tag BEFORE +/// `clean_response()` runs, so the useful text before the tag is preserved. +/// Properly closed tags (e.g. `...`) are left intact for +/// `clean_response()` to strip normally. Tags inside fenced markdown code blocks +/// or inline code spans are ignored. See issue #789. +fn truncate_at_tool_tags(text: &str) -> String { + let code_regions = find_code_regions(text); + // Use ASCII-only lowercasing so byte offsets stay valid for the original + // string. Full `to_lowercase()` can change byte lengths for non-ASCII + // chars (e.g. the Kelvin sign), making positions unreliable. + let lower = text.to_ascii_lowercase(); + let first_unclosed = TOOL_TAG_PATTERNS + .iter() + .filter_map(|p| { + let mut search_from = 0; + loop { + match lower[search_from..].find(p) { + Some(offset) => { + let pos = search_from + offset; + if is_inside_code(pos, &code_regions) { + search_from = pos + 1; + continue; + } + // Check if this tag has a matching closing tag after it. + // If so, clean_response() can handle it — skip to next. + let after_open = pos + p.len(); + if closing_tag_for(p) + .is_some_and(|close| lower[after_open..].contains(close.as_str())) + { + search_from = after_open; + continue; + } + // Unclosed tag — truncate here + return Some(pos); + } + None => return None, + } + } + }) + .min(); + match first_unclosed { + Some(pos) => { + tracing::debug!( + original_len = text.len(), + truncated_at = pos, + "Truncated response at unclosed tool-call XML tag (issue #789)" + ); + text[..pos].to_string() + } + None => text.to_string(), + } +} + +/// Derive the closing tag for a tool-call opening pattern. +/// +/// Examples: `` → ``, `<|tool_call|>` → `<|/tool_call|>`. +fn closing_tag_for(open_pattern: &str) -> Option { + if let Some(name) = open_pattern + .strip_prefix("<|") + .and_then(|s| s.strip_suffix("|>")) + { + // Pipe-delimited: <|tool_call|> → <|/tool_call|> + Some(format!("<|/{name}|>")) + } else if let Some(rest) = open_pattern.strip_prefix('<') { + // Standard XML: or + let name = rest.trim_end_matches('>').trim(); + Some(format!("")) + } else { + None + } +} + /// Strip thinking/reasoning tags using regex, respecting code regions. /// /// Strict mode: an unclosed opening tag discards all trailing text after it. @@ -2414,4 +2544,588 @@ That's my plan."#; let text = "I said let me be clear, then let me fetch the data."; assert!(llm_signals_tool_intent(text)); } + + // ---- Issue #789: truncate_at_tool_tags tests ---- + + #[test] + fn test_truncate_preserves_text_before_tool_tag() { + let input = "Here is my answer about the topic.\n{\"name\": \"search\"}"; + assert_eq!( + truncate_at_tool_tags(input), + "Here is my answer about the topic.\n" + ); + } + + #[test] + fn test_truncate_no_tool_tags_unchanged() { + let input = "Just a normal response with no tool tags."; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_empty_string() { + assert_eq!(truncate_at_tool_tags(""), ""); + } + + #[test] + fn test_truncate_tool_tag_at_start() { + assert_eq!( + truncate_at_tool_tags("{\"name\": \"search\"}"), + "" + ); + } + + #[test] + fn test_truncate_picks_earliest_unclosed_tag() { + // ... is closed — skipped. + // second is unclosed — truncated here. + let input = "Text before first and second"; + assert_eq!( + truncate_at_tool_tags(input), + "Text before first and " + ); + } + + #[test] + fn test_truncate_pipe_delimited_tags() { + let input = "Answer here\n<|tool_call|>{\"name\": \"fetch\"}"; + assert_eq!(truncate_at_tool_tags(input), "Answer here\n"); + } + + #[test] + fn test_truncate_closed_tag_with_attributes_preserved() { + // Closed tag (even with attributes) is left for clean_response() + let input = "Some text {\"name\": \"test\"}"; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_unclosed_tag_with_attributes() { + let input = "Some text {\"name\": \"test\"}"; + assert_eq!(truncate_at_tool_tags(input), "Some text "); + } + + #[test] + fn test_truncate_whitespace_only_before_tag() { + assert_eq!(truncate_at_tool_tags(" \n\n{}"), " \n\n"); + } + + #[test] + fn test_truncate_ignores_tags_inside_code_blocks() { + let input = "Here's the XML format:\n\n```xml\n{\"name\": \"search\"}\n```\n\nYou can use this to call tools."; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_finds_tag_after_code_block() { + let input = "Example:\n\n```\nexample\n```\n\nReal output:\n{\"name\": \"x\"}"; + assert_eq!( + truncate_at_tool_tags(input), + "Example:\n\n```\nexample\n```\n\nReal output:\n" + ); + } + + // ---- Issue #789: full pipeline (truncate + clean_response) tests ---- + + #[test] + fn test_issue_789_force_text_unclosed_tool_tag() { + let model_output = "The file contains a main function that initializes the server.\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"src/main.rs\"}}"; + let pre_truncated = truncate_at_tool_tags(model_output); + let cleaned = clean_response(&pre_truncated); + assert_eq!( + cleaned, + "The file contains a main function that initializes the server." + ); + } + + #[test] + fn test_issue_789_only_tool_tag_produces_empty() { + let model_output = "{\"name\": \"search\", \"arguments\": {\"q\": \"test\"}}"; + let pre_truncated = truncate_at_tool_tags(model_output); + let cleaned = clean_response(&pre_truncated); + assert!(cleaned.trim().is_empty()); + } + + #[test] + fn test_issue_789_thinking_then_tool_tag() { + let model_output = + "I should search for thisLet me help you.\n{\"name\": \"s\"}"; + let pre_truncated = truncate_at_tool_tags(model_output); + let cleaned = clean_response(&pre_truncated); + assert_eq!(cleaned, "Let me help you."); + } + + #[test] + fn test_issue_789_closed_tool_tag_preserved_for_clean_response() { + // Closed tags are left intact — clean_response() strips them normally, + // preserving any text after the tag. + let model_output = "Info here.\n{\"name\": \"x\"}\nMore text."; + let pre_truncated = truncate_at_tool_tags(model_output); + assert_eq!( + pre_truncated, model_output, + "Closed tag should not be truncated" + ); + let cleaned = clean_response(&pre_truncated); + assert_eq!(cleaned, "Info here.\n\nMore text."); + } + + // ---- Issue #789: conditional system prompt tests ---- + + fn make_reasoning_with_model(model: &str) -> Reasoning { + use crate::testing::StubLlm; + Reasoning::new(Arc::new(StubLlm::new("test"))).with_model_name(model.to_string()) + } + + #[test] + fn test_system_prompt_skips_think_final_for_native_thinking() { + let reasoning = make_reasoning_with_model("qwen3-8b"); + let prompt = reasoning.build_system_prompt_with_tools(&[]); + assert!( + !prompt.contains(""), + "Native thinking model should NOT have in system prompt" + ); + assert!(prompt.contains("Respond directly with your answer")); + } + + #[test] + fn test_system_prompt_includes_think_final_for_regular_model() { + let reasoning = make_reasoning_with_model("llama-3.1-70b"); + let prompt = reasoning.build_system_prompt_with_tools(&[]); + assert!(prompt.contains("")); + assert!(prompt.contains("")); + } + + #[test] + fn test_system_prompt_defaults_to_think_final_when_no_model() { + use crate::testing::StubLlm; + let reasoning = Reasoning::new(Arc::new(StubLlm::new("test"))); + let prompt = reasoning.build_system_prompt_with_tools(&[]); + assert!(prompt.contains("")); + assert!(prompt.contains("")); + } + + #[test] + fn test_system_prompt_deepseek_r1_skips_think_final() { + let reasoning = make_reasoning_with_model("deepseek-r1-distill-qwen-32b"); + let prompt = reasoning.build_system_prompt_with_tools(&[]); + assert!(!prompt.contains("CRITICAL")); + assert!(prompt.contains("Respond directly")); + } + + // ---- Issue #789: additional edge case tests for truncate_at_tool_tags ---- + + #[test] + fn test_truncate_unicode_content_before_tool_tag() { + let input = "こんにちは世界！素晴らしい結果です。\n{\"name\": \"search\"}"; + assert_eq!( + truncate_at_tool_tags(input), + "こんにちは世界！素晴らしい結果です。\n" + ); + } + + #[test] + fn test_truncate_emoji_content_preserved() { + let input = "The answer is 42 🎉🚀\n{\"name\": \"x\"}"; + assert_eq!(truncate_at_tool_tags(input), "The answer is 42 🎉🚀\n"); + } + + #[test] + fn test_truncate_very_long_text_before_tag() { + let long_text = "A".repeat(10_000); + let input = format!("{}\n{{\"name\": \"x\"}}", long_text); + let result = truncate_at_tool_tags(&input); + assert_eq!(result.len(), long_text.len() + 1); // +1 for \n + assert!(result.starts_with("AAAA")); + } + + #[test] + fn test_truncate_multiple_code_blocks_with_tags() { + let input = "Explanation:\n\n```python\n# in comment\nprint('hi')\n```\n\nAnd also:\n\n```xml\nexample\n```\n\nFinal answer here."; + // Both tags are inside code blocks, so nothing is truncated + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_inline_code_with_tool_tag() { + let input = "Use `` to invoke tools.\n{\"name\": \"real\"}"; + // First occurrence is in inline code, second is real + assert_eq!( + truncate_at_tool_tags(input), + "Use `` to invoke tools.\n" + ); + } + + #[test] + fn test_truncate_tag_immediately_after_code_block() { + let input = "```\nexample\n```\n{\"name\": \"x\"}"; + assert_eq!(truncate_at_tool_tags(input), "```\nexample\n```\n"); + } + + #[test] + fn test_truncate_interleaved_thinking_and_tool_tags() { + // Simulate: thinking tag + text + tool tag + let input = "reasoningHere's the answer.\n{\"name\": \"y\"}"; + let truncated = truncate_at_tool_tags(input); + let cleaned = clean_response(&truncated); + assert_eq!(cleaned, "Here's the answer."); + } + + #[test] + fn test_truncate_closed_tool_calls_plural_preserved() { + // Closed ... left for clean_response() + let input = "Answer.\n[{\"name\": \"a\"}, {\"name\": \"b\"}]"; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_unclosed_tool_calls_plural() { + let input = "Answer.\n[{\"name\": \"a\"}, {\"name\": \"b\"}]"; + assert_eq!(truncate_at_tool_tags(input), "Answer.\n"); + } + + #[test] + fn test_truncate_closed_pipe_function_call_preserved() { + let input = "Done!\n<|function_call|>{\"name\": \"x\"}<|/function_call|>"; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_unclosed_pipe_function_call() { + let input = "Done!\n<|function_call|>{\"name\": \"x\"}"; + assert_eq!(truncate_at_tool_tags(input), "Done!\n"); + } + + #[test] + fn test_truncate_adversarial_nested_code_blocks() { + // Adversarial: code block inside another structure + let input = "```\nouter\n```\n\nReal text.\n\n```\ninside\n```\n\n{\"name\": \"real\"}"; + let result = truncate_at_tool_tags(input); + assert!(result.contains("Real text.")); + assert!(!result.contains("{\"name\": \"real\"}")); + } + + // ---- Issue #789: StubLlm integration tests ---- + + #[tokio::test] + async fn test_complete_truncates_tool_tags_from_response() { + use crate::testing::StubLlm; + let response = "The server has 3 endpoints.\n{\"name\": \"read_file\"}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let request = CompletionRequest::new(vec![ChatMessage::user("describe the server")]); + let (result, _usage) = reasoning.complete(request).await.unwrap(); + assert_eq!(result, "The server has 3 endpoints."); + } + + #[tokio::test] + async fn test_complete_with_only_tool_tag_returns_empty() { + use crate::testing::StubLlm; + let response = "{\"name\": \"search\", \"arguments\": {}}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let request = CompletionRequest::new(vec![ChatMessage::user("hello")]); + let (result, _usage) = reasoning.complete(request).await.unwrap(); + assert!(result.trim().is_empty()); + } + + #[tokio::test] + async fn test_respond_with_tools_force_text_truncates_tool_tags() { + use crate::testing::StubLlm; + let response = "Here is my analysis of the code.\n{\"name\": \"read_file\", \"arguments\": {\"path\": \"main.rs\"}}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let mut context = + ReasoningContext::new().with_message(ChatMessage::user("analyze the code")); + context.force_text = true; + + let output = reasoning.respond_with_tools(&context).await.unwrap(); + match output.result { + RespondResult::Text(text) => { + assert_eq!(text, "Here is my analysis of the code."); + } + RespondResult::ToolCalls { .. } => { + panic!("Expected text result in force_text mode"); + } + } + } + + #[tokio::test] + async fn test_respond_with_tools_force_text_only_tag_uses_fallback() { + use crate::testing::StubLlm; + let response = "{\"name\": \"search\"}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let mut context = ReasoningContext::new().with_message(ChatMessage::user("hi")); + context.force_text = true; + + let output = reasoning.respond_with_tools(&context).await.unwrap(); + match output.result { + RespondResult::Text(text) => { + assert_eq!(text, "I'm not sure how to respond to that."); + } + RespondResult::ToolCalls { .. } => { + panic!("Expected fallback text, not tool calls"); + } + } + } + + #[tokio::test] + async fn test_plan_truncates_tool_tags_before_json() { + use crate::testing::StubLlm; + let response = r#"Let me plan{"goal": "Test goal", "actions": [{"tool_name": "search", "parameters": {}, "reasoning": "find files", "expected_outcome": "results"}], "confidence": 0.9} +{"name": "search"}"#; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let context = ReasoningContext::new() + .with_message(ChatMessage::user("plan a search")) + .with_job("Search for relevant files"); + + let plan = reasoning.plan(&context).await.unwrap(); + assert_eq!(plan.goal, "Test goal"); + assert!(!plan.actions.is_empty()); + } + + // ---- Issue #789: model name propagation test ---- + + #[tokio::test] + async fn test_with_model_name_affects_system_prompt() { + use crate::testing::StubLlm; + // StubLlm model_name is "stub-model" by default, but Reasoning.model_name + // is what matters for system prompt building. + let llm = Arc::new(StubLlm::new("test").with_model_name("qwen3-8b")); + let reasoning = Reasoning::new(llm.clone()).with_model_name("qwen3-8b".to_string()); + + let prompt = reasoning.build_system_prompt_with_tools(&[]); + assert!( + !prompt.contains(""), + "Qwen3 model should get native thinking system prompt" + ); + assert!(prompt.contains("Respond directly")); + + // Now create reasoning WITHOUT with_model_name — should get default prompt + let reasoning_no_model = Reasoning::new(llm); + let prompt2 = reasoning_no_model.build_system_prompt_with_tools(&[]); + assert!( + prompt2.contains(""), + "Without model name, should get default think/final prompt" + ); + } + + // ---- Issue #789: case-insensitive truncation ---- + + #[test] + fn test_truncate_case_insensitive_upper() { + let input = "Some answer.\n{\"name\": \"search\"}"; + assert_eq!(truncate_at_tool_tags(input), "Some answer.\n"); + } + + #[test] + fn test_truncate_case_insensitive_mixed() { + let input = "Result here.\n{\"name\": \"x\"}"; + assert_eq!(truncate_at_tool_tags(input), "Result here.\n"); + } + + #[test] + fn test_truncate_unicode_before_case_insensitive_tag_no_panic() { + // Regression: to_lowercase() can change byte lengths for non-ASCII chars + // (e.g. Kelvin sign U+212A is 3 bytes, lowercases to 'k' which is 1 byte). + // Using to_ascii_lowercase() keeps byte offsets stable. + let input = "Ответ: 42\n{\"name\": \"x\"}"; + assert_eq!(truncate_at_tool_tags(input), "Ответ: 42\n"); + } + + #[test] + fn test_truncate_case_insensitive_function_call_closed() { + // Closed tag (case-insensitive) preserved for clean_response() + let input = "Done.\n{\"name\": \"y\"}"; + assert_eq!(truncate_at_tool_tags(input), input); + } + + #[test] + fn test_truncate_case_insensitive_function_call_unclosed() { + let input = "Done.\n{\"name\": \"y\"}"; + assert_eq!(truncate_at_tool_tags(input), "Done.\n"); + } + + // ---- Issue #789: evaluate_success integration test ---- + + #[tokio::test] + async fn test_evaluate_success_truncates_tool_tags() { + use crate::testing::StubLlm; + let response = r#"evaluating{"success": true, "confidence": 0.85, "reasoning": "Task completed", "issues": [], "suggestions": []} +{"name": "verify"}"#; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let context = ReasoningContext::new().with_job("Test task"); + let eval = reasoning + .evaluate_success(&context, "The job is done") + .await + .unwrap(); + assert!(eval.success); + assert_eq!(eval.confidence, 0.85); + } + + // ---- Issue #789: respond_with_tools recovered tool calls path ---- + + #[tokio::test] + async fn test_respond_with_tools_recovered_tool_calls_preserves_text() { + use crate::testing::StubLlm; + // StubLlm returns empty tool_calls + content with XML tool tags. + // The recovery path should parse the tool call AND preserve text before it. + let response = "Let me search for that.\n{\"name\": \"tool_list\", \"arguments\": {}}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let context = ReasoningContext::new() + .with_message(ChatMessage::user("list tools")) + .with_tools(vec![ToolDefinition { + name: "tool_list".to_string(), + description: "Lists tools".to_string(), + parameters: serde_json::json!({}), + }]); + + let output = reasoning.respond_with_tools(&context).await.unwrap(); + match output.result { + RespondResult::ToolCalls { + tool_calls, + content, + } => { + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].name, "tool_list"); + // Text before the tag should be preserved + assert_eq!(content.as_deref(), Some("Let me search for that.")); + } + RespondResult::Text(_) => { + panic!("Expected recovered tool calls, got text"); + } + } + } + + #[tokio::test] + async fn test_respond_with_tools_recovered_only_tag_content_is_none() { + use crate::testing::StubLlm; + // Content is ONLY a tool call tag — after truncation+cleaning, content should be None + let response = "{\"name\": \"tool_list\", \"arguments\": {}}"; + let llm = Arc::new(StubLlm::new(response)); + let reasoning = Reasoning::new(llm); + + let context = ReasoningContext::new() + .with_message(ChatMessage::user("list tools")) + .with_tools(vec![ToolDefinition { + name: "tool_list".to_string(), + description: "Lists tools".to_string(), + parameters: serde_json::json!({}), + }]); + + let output = reasoning.respond_with_tools(&context).await.unwrap(); + match output.result { + RespondResult::ToolCalls { + tool_calls, + content, + } => { + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].name, "tool_list"); + assert!( + content.is_none(), + "Content should be None when only tool tags present" + ); + } + RespondResult::Text(_) => { + panic!("Expected recovered tool calls, got text"); + } + } + } + + // ---- Issue #789: OpenAI reasoning models negative test ---- + + #[test] + fn test_openai_reasoning_models_not_detected() { + use crate::llm::reasoning_models::has_native_thinking; + assert!(!has_native_thinking("o1")); + assert!(!has_native_thinking("o1-mini")); + assert!(!has_native_thinking("o1-preview")); + assert!(!has_native_thinking("o3-mini")); + assert!(!has_native_thinking("o4-mini")); + } + + // ---- closing_tag_for() unit tests ---- + + #[test] + fn test_closing_tag_for_standard_tags() { + assert_eq!( + closing_tag_for("").as_deref(), + Some("") + ); + assert_eq!( + closing_tag_for("").as_deref(), + Some("") + ); + assert_eq!( + closing_tag_for("").as_deref(), + Some("") + ); + } + + #[test] + fn test_closing_tag_for_space_suffixed_patterns() { + // Patterns with trailing space (for attribute matching) + assert_eq!( + closing_tag_for("") + ); + assert_eq!( + closing_tag_for("") + ); + assert_eq!( + closing_tag_for("") + ); + } + + #[test] + fn test_closing_tag_for_pipe_delimited() { + assert_eq!( + closing_tag_for("<|tool_call|>").as_deref(), + Some("<|/tool_call|>") + ); + assert_eq!( + closing_tag_for("<|function_call|>").as_deref(), + Some("<|/function_call|>") + ); + assert_eq!( + closing_tag_for("<|tool_calls|>").as_deref(), + Some("<|/tool_calls|>") + ); + } + + #[test] + fn test_closing_tag_for_covers_all_patterns() { + // Every entry in TOOL_TAG_PATTERNS must produce a closing tag + for pattern in TOOL_TAG_PATTERNS { + assert!( + closing_tag_for(pattern).is_some(), + "closing_tag_for({:?}) returned None", + pattern + ); + } + } + + // ---- truncation with multiple tags: first closed, second unclosed ---- + + #[test] + fn test_truncate_mixed_closed_then_unclosed_different_types() { + let input = "Text {} middle {\"name\": \"x\"}"; + // function_call is closed → skipped. tool_call is unclosed → truncated. + assert_eq!( + truncate_at_tool_tags(input), + "Text {} middle " + ); + } } diff --git a/src/llm/reasoning_models.rs b/src/llm/reasoning_models.rs new file mode 100644 index 00000000..307cb0a3 --- /dev/null +++ b/src/llm/reasoning_models.rs @@ -0,0 +1,134 @@ +//! Reasoning/thinking model detection utilities. +//! +//! Models with native thinking support produce structured chain-of-thought +//! via `reasoning_content` fields or built-in `` tags. Injecting +//! IronClaw's own `/` format instructions into the system +//! prompt collides with these models' native behavior, causing: +//! - Thinking-only responses with no visible content +//! - Double-wrapped thinking tags that confuse response cleaning +//! +//! When a model has native thinking, we skip the `/` prompt +//! injection and let the model use its own format. The response cleaning +//! pipeline already handles stripping all known thinking tag variants. +//! +//! ## Design note: why match broadly (e.g. all Qwen3)? +//! +//! Some families (Qwen3) have ALL variants trained with native `` tags, +//! even tiny models like 0.6B. Thinking can be disabled at inference time via +//! `enable_thinking=false`, but we can't detect that from the model name alone. +//! We err on the safe side: skip injection for all variants because: +//! - False negative (inject when model thinks natively) = broken responses +//! - False positive (skip injection for non-thinking model) = less structured +//! but working responses +//! +//! For families where only SOME variants reason (GLM-4), we match specific +//! sub-families (glm-z1, glm-4-plus) to avoid false positives. + +/// Known model families with native thinking/reasoning support. +/// +/// These models produce chain-of-thought reasoning either via a dedicated +/// `reasoning_content` response field or via built-in `` tags that +/// the model was trained to emit without prompt injection. +const NATIVE_THINKING_PATTERNS: &[&str] = &[ + // Qwen3 family — ALL variants (0.6B through 235B) emit native tags + // by default. Thinking can be toggled via `enable_thinking` parameter or + // `/think` `/no_think` soft switches, but the default is ON and we can't + // detect the runtime setting from the model name. + "qwen3", + // QwQ is Qwen's dedicated reasoning model (based on Qwen2.5-32B + RL). + // Always thinks, no disable toggle. + "qwq", + // DeepSeek reasoning models — native reasoning_content field + "deepseek-r1", + "deepseek-reasoner", + // GLM reasoning variants only (glm-4-flash, glm-4-air, glm-4v do NOT reason) + "glm-z1", + "glm-4-plus", + "glm-5", + // Nanbeige reasoning models + "nanbeige", + // Step reasoning models (3.5+ have native thinking; step-3 base does not) + "step-3.5", + // MiniMax reasoning models + "minimax-m2", +]; + +/// Check if a model name indicates native thinking/reasoning support. +/// +/// Models that return `true` should NOT have IronClaw's `/` +/// format instructions injected into their system prompt, as this collides +/// with their built-in reasoning behavior. +/// +/// Note: this is a best-effort heuristic based on model name. Some models +/// support toggling thinking at runtime (e.g. Qwen3's `enable_thinking`), +/// which we cannot detect here. We default to assuming thinking is ON for +/// models that have it, since that's the default behavior. +pub fn has_native_thinking(model: &str) -> bool { + let lower = model.to_ascii_lowercase(); + NATIVE_THINKING_PATTERNS.iter().any(|p| lower.contains(p)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_qwen3_models() { + // All Qwen3 variants have native thinking (even small ones) + assert!(has_native_thinking("qwen3-coder-next-80b")); + assert!(has_native_thinking("Qwen3.5-35B")); + assert!(has_native_thinking("qwen3-0.6b")); + assert!(has_native_thinking("qwen3:8b")); + assert!(has_native_thinking("qwen3-30b-a3b")); + // Ollama-style tag format + assert!(has_native_thinking("qwen3-coder:latest")); + } + + #[test] + fn detects_qwq() { + assert!(has_native_thinking("qwq-32b")); + assert!(has_native_thinking("QwQ-32B-Preview")); + } + + #[test] + fn detects_deepseek_reasoning() { + assert!(has_native_thinking("deepseek-r1-distill-qwen-32b")); + assert!(has_native_thinking("deepseek-reasoner")); + } + + #[test] + fn detects_glm_reasoning_variants() { + assert!(has_native_thinking("glm-z1-airx")); + assert!(has_native_thinking("glm-4-plus")); + assert!(has_native_thinking("GLM-5")); + } + + #[test] + fn detects_other_reasoning_models() { + assert!(has_native_thinking("nanbeige-4.1-3b")); + assert!(has_native_thinking("step-3.5-flash-197b")); + assert!(has_native_thinking("minimax-m2.5-139b")); + } + + #[test] + fn rejects_non_reasoning_models() { + assert!(!has_native_thinking("gpt-4o")); + assert!(!has_native_thinking("claude-3-5-sonnet")); + assert!(!has_native_thinking("llama-3.1-70b")); + assert!(!has_native_thinking("mistral-7b")); + assert!(!has_native_thinking("gemini-2.0-flash")); + } + + #[test] + fn rejects_non_reasoning_variants_in_same_family() { + // Qwen2.5 does NOT have native thinking (only Qwen3/QwQ do) + assert!(!has_native_thinking("qwen2.5:7b")); + assert!(!has_native_thinking("qwen2.5-instruct")); + // GLM-4 base variants do NOT have reasoning_content + assert!(!has_native_thinking("glm-4-flash")); + assert!(!has_native_thinking("glm-4-air")); + assert!(!has_native_thinking("glm-4v")); + // step-3 base does not reason (only 3.5+) + assert!(!has_native_thinking("step-3-mini")); + } +} diff --git a/src/llm/registry.rs b/src/llm/registry.rs index 36cb7001..434c698a 100644 --- a/src/llm/registry.rs +++ b/src/llm/registry.rs @@ -113,6 +113,33 @@ impl SetupHint { } } +/// Validates unsupported_params during deserialization. +/// +/// Only allows: "temperature", "max_tokens", "stop_sequences". +/// Invalid parameter names cause a deserialization error. +mod unsupported_params_de { + use serde::{Deserialize, Deserializer}; + + const VALID_PARAMS: &[&str] = &["temperature", "max_tokens", "stop_sequences"]; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let params: Vec = Deserialize::deserialize(deserializer)?; + for param in ¶ms { + if !VALID_PARAMS.contains(¶m.as_str()) { + return Err(serde::de::Error::custom(format!( + "unsupported parameter name '{}': must be one of: {}", + param, + VALID_PARAMS.join(", ") + ))); + } + } + Ok(params) + } +} + /// Declarative definition of an LLM provider. /// /// One JSON object in `providers.json` maps to one `ProviderDefinition`. @@ -155,7 +182,8 @@ pub struct ProviderDefinition { /// Parameter names that this provider does not support (e.g., `["temperature"]`). /// Supported keys: `"temperature"`, `"max_tokens"`, `"stop_sequences"`. /// Listed parameters are stripped from requests before sending to avoid 400 errors. - #[serde(default)] + /// Invalid parameter names cause a deserialization error. + #[serde(default, deserialize_with = "unsupported_params_de::deserialize")] pub unsupported_params: Vec, } @@ -752,7 +780,8 @@ mod tests { "groq should have empty unsupported_params (field absent in JSON)" ); - // Every non-empty entry should contain valid param names + // All entries should only contain valid param names + // (Invalid names should be rejected at deserialization time) for def in &providers { for param in &def.unsupported_params { assert!( @@ -760,10 +789,42 @@ mod tests { "{}: unsupported_params contains empty string", def.id ); + assert!( + matches!( + param.as_str(), + "temperature" | "max_tokens" | "stop_sequences" + ), + "{}: unsupported_params contains invalid parameter '{}'", + def.id, + param + ); } } } + #[test] + fn test_unsupported_params_validation_rejects_invalid() { + // Invalid parameter names should cause deserialization error + let invalid_json = r#"[{ + "id": "test", + "protocol": "open_ai_completions", + "model_env": "TEST_MODEL", + "default_model": "test-model", + "description": "Test provider", + "unsupported_params": ["temperrature"] + }]"#; + + let result: Result, _> = serde_json::from_str(invalid_json); + assert!( + result.is_err(), + "should reject invalid parameter name 'temperrature'" + ); + assert!( + result.err().unwrap().to_string().contains("temperrature"), + "error message should mention the invalid parameter" + ); + } + #[test] fn test_all_builtin_api_key_providers_have_api_key_env() { // Every built-in provider with SetupHint::ApiKey must have api_key_env diff --git a/src/llm/response_cache.rs b/src/llm/response_cache.rs index b1e7aa8e..b8238427 100644 --- a/src/llm/response_cache.rs +++ b/src/llm/response_cache.rs @@ -205,7 +205,7 @@ impl LlmProvider for CachedProvider { let hit_count = entry.hit_count; // Clone now so we can release the mutable borrow before stats. let cached_response = entry.response.clone(); - tracing::debug!(hits = hit_count, "response cache hit"); + tracing::trace!(hits = hit_count, "response cache hit"); // Drop the mutable borrow of `entry` before reading `guard` immutably. let _ = entry; let total_hits = self.total_hit_count.fetch_add(1, Ordering::Relaxed) + 1; diff --git a/src/llm/rig_adapter.rs b/src/llm/rig_adapter.rs index 5b835536..41724c31 100644 --- a/src/llm/rig_adapter.rs +++ b/src/llm/rig_adapter.rs @@ -28,7 +28,8 @@ use crate::llm::error::LlmError; use crate::llm::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCall as IronToolCall, ToolCompletionRequest, ToolCompletionResponse, - ToolDefinition as IronToolDefinition, + ToolDefinition as IronToolDefinition, strip_unsupported_completion_params, + strip_unsupported_tool_params, }; /// Adapter that wraps a rig-core `CompletionModel` and implements `LlmProvider`. @@ -100,31 +101,12 @@ impl RigAdapter { /// Strip unsupported fields from a `CompletionRequest` in place. fn strip_unsupported_completion_params(&self, req: &mut CompletionRequest) { - if self.unsupported_params.is_empty() { - return; - } - if self.unsupported_params.contains("temperature") { - req.temperature = None; - } - if self.unsupported_params.contains("max_tokens") { - req.max_tokens = None; - } - if self.unsupported_params.contains("stop_sequences") { - req.stop_sequences = None; - } + strip_unsupported_completion_params(&self.unsupported_params, req); } /// Strip unsupported fields from a `ToolCompletionRequest` in place. fn strip_unsupported_tool_params(&self, req: &mut ToolCompletionRequest) { - if self.unsupported_params.is_empty() { - return; - } - if self.unsupported_params.contains("temperature") { - req.temperature = None; - } - if self.unsupported_params.contains("max_tokens") { - req.max_tokens = None; - } + strip_unsupported_tool_params(&self.unsupported_params, req); } } diff --git a/src/llm/session.rs b/src/llm/session.rs index 3d1c4785..1cb858a1 100644 --- a/src/llm/session.rs +++ b/src/llm/session.rs @@ -627,6 +627,9 @@ pub async fn create_session_manager(config: SessionConfig) -> Arc { - tracing::debug!( + tracing::trace!( model = %self.cheap.model_name(), "Smart routing: Simple task -> cheap model" ); @@ -880,7 +880,7 @@ impl LlmProvider for SmartRoutingProvider { self.cheap.complete(request).await } TaskComplexity::Complex => { - tracing::debug!( + tracing::trace!( model = %self.primary.model_name(), "Smart routing: Complex task -> primary model" ); @@ -889,7 +889,7 @@ impl LlmProvider for SmartRoutingProvider { } TaskComplexity::Moderate => { if self.config.cascade_enabled { - tracing::debug!( + tracing::trace!( model = %self.cheap.model_name(), "Smart routing: Moderate task -> cheap model (cascade enabled)" ); @@ -913,7 +913,7 @@ impl LlmProvider for SmartRoutingProvider { } } else { // Without cascade, moderate tasks go to cheap model - tracing::debug!( + tracing::trace!( model = %self.cheap.model_name(), "Smart routing: Moderate task -> cheap model (cascade disabled)" ); @@ -931,7 +931,7 @@ impl LlmProvider for SmartRoutingProvider { ) -> Result { self.stats.total_requests.fetch_add(1, Ordering::Relaxed); self.stats.primary_requests.fetch_add(1, Ordering::Relaxed); - tracing::debug!( + tracing::trace!( model = %self.primary.model_name(), "Smart routing: Tool use -> primary model (always)" ); diff --git a/src/main.rs b/src/main.rs index 120fa33c..1224f295 100644 --- a/src/main.rs +++ b/src/main.rs @@ -322,10 +322,16 @@ async fn async_main() -> anyhow::Result<()> { // Add HTTP channel if configured and not CLI-only mode. let mut webhook_server_addr: Option = None; + #[cfg(unix)] + let mut http_channel_state: Option> = None; if !cli.cli_only && let Some(ref http_config) = config.channels.http { let http_channel = HttpChannel::new(http_config.clone()); + #[cfg(unix)] + { + http_channel_state = Some(http_channel.shared_state()); + } webhook_routes.push(http_channel.routes()); let (host, port) = http_channel.addr(); webhook_server_addr = Some( @@ -343,7 +349,9 @@ async fn async_main() -> anyhow::Result<()> { } // Start the unified webhook server if any routes were registered. - let mut webhook_server = if !webhook_routes.is_empty() { + let webhook_server: Option>> = if !webhook_routes + .is_empty() + { let addr = webhook_server_addr.unwrap_or_else(|| std::net::SocketAddr::from(([0, 0, 0, 0], 8080))); if addr.ip().is_unspecified() { @@ -358,7 +366,7 @@ async fn async_main() -> anyhow::Result<()> { server.add_routes(routes); } server.start().await?; - Some(server) + Some(Arc::new(tokio::sync::Mutex::new(server))) } else { None }; @@ -601,6 +609,13 @@ async fn async_main() -> anyhow::Result<()> { // Clone context_manager for the reaper before it's moved into Agent::new() let reaper_context_manager = Arc::clone(&components.context_manager); + // Capture db reference for SIGHUP handler before it's moved into AgentDeps (Unix only) + #[cfg(unix)] + let sighup_settings_store: Option> = components + .db + .as_ref() + .map(|db| Arc::clone(db) as Arc); + let deps = AgentDeps { store: components.db, llm: components.llm, @@ -661,10 +676,157 @@ async fn async_main() -> anyhow::Result<()> { agent.set_routine_engine_slot(slot); } + // Prepare SIGHUP handler for hot-reloading HTTP webhook config + // Broadcast channel for clean shutdown of background tasks + let (shutdown_tx, _) = tokio::sync::broadcast::channel::<()>(1); + + #[cfg(unix)] + { + use ironclaw::channels::ChannelSecretUpdater; + // Collect all channels that support secret updates + let mut secret_updaters: Vec> = Vec::new(); + if let Some(ref state) = http_channel_state { + secret_updaters.push(Arc::clone(state) as Arc); + } + + let sighup_webhook_server = webhook_server.clone(); + let sighup_settings_store_clone = sighup_settings_store.clone(); + let sighup_secrets_store = components.secrets_store.clone(); + let mut shutdown_rx = shutdown_tx.subscribe(); + + tokio::spawn(async move { + use tokio::signal::unix::{SignalKind, signal}; + let mut sighup = match signal(SignalKind::hangup()) { + Ok(s) => s, + Err(e) => { + tracing::warn!("Failed to register SIGHUP handler: {}", e); + return; + } + }; + + loop { + // Exit loop on shutdown signal or when SIGHUP is received + tokio::select! { + _ = shutdown_rx.recv() => { + tracing::debug!("SIGHUP handler shutting down"); + break; + } + _ = sighup.recv() => { + // Handle SIGHUP signal + } + } + tracing::info!("SIGHUP received — reloading HTTP webhook config"); + + // Inject channel secrets from database into thread-safe overlay + // (similar to inject_llm_keys_from_secrets for LLM providers) + if let Some(ref secrets_store) = sighup_secrets_store { + // Inject HTTP webhook secret from encrypted store + if let Ok(webhook_secret) = secrets_store + .get_decrypted("default", "http_webhook_secret") + .await + { + // Thread-safe: Uses INJECTED_VARS mutex instead of unsafe std::env::set_var + // Config::from_env() will read from the overlay via optional_env() + ironclaw::config::inject_single_var( + "HTTP_WEBHOOK_SECRET", + webhook_secret.expose(), + ); + tracing::debug!("Injected HTTP_WEBHOOK_SECRET from secrets store"); + } + } + + // Reload config (now with secrets injected into environment) + let new_config = match &sighup_settings_store_clone { + Some(store) => { + ironclaw::config::Config::from_db(store.as_ref(), "default").await + } + None => ironclaw::config::Config::from_env().await, + }; + + let new_config = match new_config { + Ok(c) => c, + Err(e) => { + tracing::error!("SIGHUP config reload failed: {}", e); + continue; + } + }; + + let new_http = match new_config.channels.http { + Some(c) => c, + None => { + tracing::warn!("SIGHUP: HTTP channel no longer configured, skipping"); + continue; + } + }; + + // Compute new socket addr + let new_addr: std::net::SocketAddr = + match format!("{}:{}", new_http.host, new_http.port).parse() { + Ok(a) => a, + Err(e) => { + tracing::error!("SIGHUP: invalid addr in config: {}", e); + continue; + } + }; + + // Restart listener if addr changed. + // Minimize lock scope: acquire, read old addr, release, then restart. + let mut restart_failed = false; + if let Some(ref ws_arc) = sighup_webhook_server { + let old_addr = { + let ws = ws_arc.lock().await; + ws.current_addr() + }; // Lock released here + + if old_addr != new_addr { + tracing::info!( + "SIGHUP: HTTP addr {} -> {}, restarting listener", + old_addr, + new_addr + ); + // NOTE: Lock is held across restart_with_addr().await. This is + // acceptable because SIGHUP is infrequent and restart is fast. A full + // fix would require refactoring restart_with_addr to separate state + // mutation from async I/O. + let mut ws = ws_arc.lock().await; + match ws.restart_with_addr(new_addr).await { + Ok(()) => { + tracing::info!("SIGHUP: webhook server restarted on {}", new_addr); + } + Err(e) => { + tracing::error!("SIGHUP: listener restart failed: {}", e); + restart_failed = true; + } + } + } else { + tracing::debug!("SIGHUP: addr unchanged ({})", old_addr); + } + } + + // Update secrets in all configured channels (if restart succeeded or wasn't needed) + if !restart_failed { + use secrecy::{ExposeSecret, SecretString}; + let new_secret = new_http + .webhook_secret + .as_ref() + .map(|s| SecretString::from(s.expose_secret().to_string())); + + // Update all channels that support secret swapping + for updater in &secret_updaters { + updater.update_secret(new_secret.clone()).await; + } + } + } + }); + } + agent.run().await?; // ── Shutdown ──────────────────────────────────────────────────────── + // Signal background tasks (SIGHUP handler, etc.) to gracefully shut down + let _ = shutdown_tx.send(()); + // Shut down all stdio MCP server child processes. components.mcp_process_manager.shutdown_all().await; @@ -675,8 +837,8 @@ async fn async_main() -> anyhow::Result<()> { tracing::warn!("Failed to write LLM trace: {}", e); } - if let Some(ref mut server) = webhook_server { - server.shutdown().await; + if let Some(ref ws_arc) = webhook_server { + ws_arc.lock().await.shutdown().await; } if let Some(tunnel) = active_tunnel { diff --git a/src/orchestrator/api.rs b/src/orchestrator/api.rs index 82783a64..80e09073 100644 --- a/src/orchestrator/api.rs +++ b/src/orchestrator/api.rs @@ -661,12 +661,9 @@ mod tests { #[tokio::test] async fn credentials_returns_secrets_when_store_configured() { + use crate::testing::credentials::test_secrets_store; use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new( - crate::secrets::SecretsCrypto::new(SecretString::from(key.to_string())).unwrap(), - ); - let secrets_store = Arc::new(crate::secrets::InMemorySecretsStore::new(crypto)); + let secrets_store = Arc::new(test_secrets_store()); // Create a secret secrets_store diff --git a/src/registry/installer.rs b/src/registry/installer.rs index e4ae785c..342af326 100644 --- a/src/registry/installer.rs +++ b/src/registry/installer.rs @@ -20,16 +20,22 @@ const ALLOWED_ARTIFACT_HOSTS: &[&str] = &[ ]; fn should_attempt_source_fallback(err: &RegistryError) -> bool { - // MissingChecksum is intentionally allowed here — it's a bootstrapping issue - // (no release has populated checksums yet), not a security concern. Source - // builds use local trusted code. ChecksumMismatch (tampered artifact) and - // InvalidManifest (structural problem) remain blocked. - !matches!( - err, - RegistryError::AlreadyInstalled { .. } - | RegistryError::ChecksumMismatch { .. } - | RegistryError::InvalidManifest { .. } - ) + match err { + // `releases/latest` is a moving target: every new release rebuilds WASM + // extensions, so a mismatch against a `latest` URL just means the binary + // was compiled against an older release's checksum. Not a security concern + // — fall back to building from source. + // + // Version-pinned URLs (`releases/download/vX.Y.Z/`) point to an immutable + // asset; a mismatch there is genuinely suspicious and remains a hard block. + RegistryError::ChecksumMismatch { url, .. } => { + url.contains("github.com/nearai/ironclaw/releases/latest/") + } + // Never fall back for these — they signal a structural problem or a + // deliberate "already done" state, not a transient artifact issue. + RegistryError::AlreadyInstalled { .. } | RegistryError::InvalidManifest { .. } => false, + _ => true, + } } fn is_allowed_artifact_host(host: &str) -> bool { @@ -931,14 +937,6 @@ mod tests { }; assert!(!should_attempt_source_fallback(&already)); - let checksum = RegistryError::ChecksumMismatch { - url: "https://github.com/nearai/ironclaw/releases/latest/download/demo.wasm" - .to_string(), - expected_sha256: "deadbeef".to_string(), - actual_sha256: "feedface".to_string(), - }; - assert!(!should_attempt_source_fallback(&checksum)); - let invalid = RegistryError::InvalidManifest { name: "demo".to_string(), field: "artifacts.wasm32-wasip2.url", @@ -1088,4 +1086,30 @@ mod tests { assert!(result.is_err()); } + + // Regression test for issue #439: ChecksumMismatch on a `releases/latest` URL + // must allow source-build fallback (moving-target URL, not a security concern), + // while a mismatch on a version-pinned URL must remain a hard block. + #[test] + fn test_source_fallback_on_latest_url_mismatch() { + let latest_mismatch = RegistryError::ChecksumMismatch { + url: "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz".to_string(), + expected_sha256: "aaa".to_string(), + actual_sha256: "bbb".to_string(), + }; + assert!( + should_attempt_source_fallback(&latest_mismatch), + "ChecksumMismatch on releases/latest URL should allow source fallback" + ); + + let pinned_mismatch = RegistryError::ChecksumMismatch { + url: "https://github.com/nearai/ironclaw/releases/download/v0.7.0/github-0.2.0-wasm32-wasip2.tar.gz".to_string(), + expected_sha256: "aaa".to_string(), + actual_sha256: "bbb".to_string(), + }; + assert!( + !should_attempt_source_fallback(&pinned_mismatch), + "ChecksumMismatch on version-pinned URL must remain a hard block" + ); + } } diff --git a/src/safety/mod.rs b/src/safety/mod.rs index 50167fc0..e9027792 100644 --- a/src/safety/mod.rs +++ b/src/safety/mod.rs @@ -164,7 +164,7 @@ impl SafetyLayer { "\n{}\n", escape_xml_attr(tool_name), sanitized, - escape_xml_content(content) + content ) } @@ -213,13 +213,6 @@ fn escape_xml_attr(s: &str) -> String { .replace('>', ">") } -/// Escape XML content. -fn escape_xml_content(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") -} - #[cfg(test)] mod tests { use super::*; @@ -235,7 +228,7 @@ mod tests { let wrapped = safety.wrap_for_llm("test_tool", "Hello ", true); assert!(wrapped.contains("name=\"test_tool\"")); assert!(wrapped.contains("sanitized=\"true\"")); - assert!(wrapped.contains("Hello <world>")); + assert!(wrapped.contains("Hello ")); } #[test] diff --git a/src/safety/validator.rs b/src/safety/validator.rs index c56789ea..d41ccc1f 100644 --- a/src/safety/validator.rs +++ b/src/safety/validator.rs @@ -117,8 +117,6 @@ impl Validator { /// Validate input text. pub fn validate(&self, input: &str) -> ValidationResult { - let mut result = ValidationResult::ok(); - // Check empty if input.is_empty() { return ValidationResult::error(ValidationError { @@ -128,10 +126,16 @@ impl Validator { }); } + self.validate_non_empty_input(input, "input") + } + + fn validate_non_empty_input(&self, input: &str, field: &str) -> ValidationResult { + let mut result = ValidationResult::ok(); + // Check length if input.len() > self.max_length { result = result.merge(ValidationResult::error(ValidationError { - field: "input".to_string(), + field: field.to_string(), message: format!( "Input too long: {} bytes (max {})", input.len(), @@ -143,7 +147,7 @@ impl Validator { if input.len() < self.min_length { result = result.merge(ValidationResult::error(ValidationError { - field: "input".to_string(), + field: field.to_string(), message: format!( "Input too short: {} bytes (min {})", input.len(), @@ -156,7 +160,7 @@ impl Validator { // Check for valid UTF-8 (should always pass since we have a &str, but check for weird chars) if input.chars().any(|c| c == '\x00') { result = result.merge(ValidationResult::error(ValidationError { - field: "input".to_string(), + field: field.to_string(), message: "Input contains null bytes".to_string(), code: ValidationErrorCode::InvalidEncoding, })); @@ -167,7 +171,7 @@ impl Validator { for pattern in &self.forbidden_patterns { if lower_input.contains(pattern) { result = result.merge(ValidationResult::error(ValidationError { - field: "input".to_string(), + field: field.to_string(), message: format!("Input contains forbidden pattern: {}", pattern), code: ValidationErrorCode::ForbiddenContent, })); @@ -196,29 +200,40 @@ impl Validator { // Recursively check all string values in the JSON fn check_strings( value: &serde_json::Value, + path: &str, validator: &Validator, result: &mut ValidationResult, ) { match value { serde_json::Value::String(s) => { - let string_result = validator.validate(s); + let string_result = if s.is_empty() { + ValidationResult::ok() + } else { + validator.validate_non_empty_input(s, path) + }; *result = std::mem::take(result).merge(string_result); } serde_json::Value::Array(arr) => { - for item in arr { - check_strings(item, validator, result); + for (i, item) in arr.iter().enumerate() { + let child_path = format!("{path}[{i}]"); + check_strings(item, &child_path, validator, result); } } serde_json::Value::Object(obj) => { - for (_, v) in obj { - check_strings(v, validator, result); + for (k, v) in obj { + let child_path = if path.is_empty() { + k.clone() + } else { + format!("{path}.{k}") + }; + check_strings(v, &child_path, validator, result); } } _ => {} } } - check_strings(params, self, &mut result); + check_strings(params, "", self, &mut result); result } } @@ -312,4 +327,100 @@ mod tests { assert!(result.is_valid); // Still valid, just a warning assert!(!result.warnings.is_empty()); } + + #[test] + fn test_tool_params_allow_empty_strings() { + let validator = Validator::new(); + let result = validator.validate_tool_params(&serde_json::json!({ + "path": "", + "nested": { + "label": "" + }, + "items": [""] + })); + + assert!(result.is_valid); + assert!(result.errors.is_empty()); + } + + #[test] + fn test_tool_params_still_block_null_bytes() { + let validator = Validator::new(); + let result = validator.validate_tool_params(&serde_json::json!({ + "path": "bad\u{0000}path" + })); + + assert!(!result.is_valid); + assert!( + result + .errors + .iter() + .any(|e| e.code == ValidationErrorCode::InvalidEncoding) + ); + } + + #[test] + fn test_tool_params_still_block_forbidden_patterns() { + let validator = Validator::new().forbid_pattern("forbidden"); + let result = validator.validate_tool_params(&serde_json::json!({ + "path": "contains forbidden content" + })); + + assert!(!result.is_valid); + assert!( + result + .errors + .iter() + .any(|e| e.code == ValidationErrorCode::ForbiddenContent) + ); + } + + #[test] + fn test_tool_params_still_warn_on_repetition() { + let validator = Validator::new(); + let result = validator.validate_tool_params(&serde_json::json!({ + "content": format!("prefix{}suffix", "x".repeat(50)) + })); + + assert!(result.is_valid); + assert!( + result.warnings.iter().any(|w| w.contains("repetition")), + "expected repetition warning for tool params, got: {:?}", + result.warnings + ); + } + + #[test] + fn test_tool_params_still_warn_on_whitespace_ratio() { + let validator = Validator::new(); + // >100 chars, >90% whitespace + let result = validator.validate_tool_params(&serde_json::json!({ + "content": format!("a{}b", " ".repeat(200)) + })); + + assert!(result.is_valid); + assert!( + result.warnings.iter().any(|w| w.contains("whitespace")), + "expected whitespace warning for tool params, got: {:?}", + result.warnings + ); + } + + #[test] + fn test_tool_params_error_field_contains_json_path() { + let validator = Validator::new().forbid_pattern("evil"); + let result = validator.validate_tool_params(&serde_json::json!({ + "metadata": { + "tags": ["good", "evil"] + } + })); + + assert!(!result.is_valid); + let error = result + .errors + .iter() + .find(|e| e.code == ValidationErrorCode::ForbiddenContent) + .expect("expected forbidden content error"); + assert_eq!(error.field, "metadata.tags[1]"); + } } diff --git a/src/secrets/crypto.rs b/src/secrets/crypto.rs index 2f2de093..5d658882 100644 --- a/src/secrets/crypto.rs +++ b/src/secrets/crypto.rs @@ -153,11 +153,11 @@ mod tests { use secrecy::SecretString; use crate::secrets::crypto::SecretsCrypto; + use crate::testing::credentials::TEST_CRYPTO_KEY; fn test_crypto() -> SecretsCrypto { // 32-byte test key - let key = "0123456789abcdef0123456789abcdef"; - SecretsCrypto::new(SecretString::from(key.to_string())).unwrap() + SecretsCrypto::new(SecretString::from(TEST_CRYPTO_KEY.to_string())).unwrap() } #[test] diff --git a/src/secrets/store.rs b/src/secrets/store.rs index 0bc180a7..d98e0cca 100644 --- a/src/secrets/store.rs +++ b/src/secrets/store.rs @@ -802,30 +802,25 @@ pub mod in_memory { #[cfg(test)] mod tests { - use std::sync::Arc; - - use secrecy::SecretString; - - use crate::secrets::crypto::SecretsCrypto; use crate::secrets::store::SecretsStore; - use crate::secrets::store::in_memory::InMemorySecretsStore; use crate::secrets::types::CreateSecretParams; + use crate::testing::credentials::{ + TEST_OPENAI_API_KEY_SHORT, TEST_SECRET_VALUE, TEST_STRIPE_KEY, test_secrets_store, + }; - fn test_store() -> InMemorySecretsStore { - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - InMemorySecretsStore::new(crypto) + fn test_store() -> crate::secrets::store::in_memory::InMemorySecretsStore { + test_secrets_store() } #[tokio::test] async fn test_create_and_get() { let store = test_store(); - let params = CreateSecretParams::new("api_key", "sk-test-12345"); + let params = CreateSecretParams::new("api_key", TEST_SECRET_VALUE); store.create("user1", params).await.unwrap(); let decrypted = store.get_decrypted("user1", "api_key").await.unwrap(); - assert_eq!(decrypted.expose(), "sk-test-12345"); + assert_eq!(decrypted.expose(), TEST_SECRET_VALUE); } #[tokio::test] @@ -878,11 +873,17 @@ mod tests { async fn test_is_accessible() { let store = test_store(); store - .create("user1", CreateSecretParams::new("openai_key", "sk-test")) + .create( + "user1", + CreateSecretParams::new("openai_key", TEST_OPENAI_API_KEY_SHORT), + ) .await .unwrap(); store - .create("user1", CreateSecretParams::new("stripe_key", "sk-live")) + .create( + "user1", + CreateSecretParams::new("stripe_key", TEST_STRIPE_KEY), + ) .await .unwrap(); diff --git a/src/setup/README.md b/src/setup/README.md index b94b3d0b..4b734e36 100644 --- a/src/setup/README.md +++ b/src/setup/README.md @@ -209,19 +209,18 @@ env-var mode or skipped secrets. | Anthropic | API key | `anthropic_api_key` | `ANTHROPIC_API_KEY` | | OpenAI | API key | `openai_api_key` | `OPENAI_API_KEY` | | Ollama | None | - | - | -| OpenRouter¹ | API key | `llm_compatible_api_key` | `LLM_API_KEY` | -| OpenAI-compatible¹ | Optional API key | `llm_compatible_api_key` | `LLM_API_KEY` | +| OpenRouter | API key | `llm_openrouter_api_key` | `OPENROUTER_API_KEY` | +| OpenAI-compatible | Optional API key | `llm_compatible_api_key` | `LLM_API_KEY` | | AWS Bedrock | AWS credentials (IAM, SSO, instance roles) | - | - | -¹ OpenRouter and OpenAI-compatible share the same secret name and env var because -OpenRouter is stored as `llm_backend = "openai_compatible"` under the hood. -Switching between them overwrites the same credential slot. +**OpenRouter** is a standalone registry provider (`providers.json` id `"openrouter"`) +with its own secret name and env var. It is **not** stored as `openai_compatible`. -**OpenRouter** (`setup_openrouter`): -- Pre-configured OpenAI-compatible preset with base URL `https://openrouter.ai/api/v1` -- Delegates to `setup_api_key_provider()` with a display name override ("OpenRouter") -- Sets `llm_backend = "openai_compatible"` and `openai_compatible_base_url` automatically -- Clears `selected_model` so Step 4 prompts for a model name (manual text input, no API-based model fetching) +**OpenRouter** (`setup.kind = "api_key"` in `providers.json`): +- Standalone provider with base URL `https://openrouter.ai/api/v1` +- Delegates to `setup_api_key_provider()` with display name "OpenRouter" +- API key is required (`api_key_required: true`) +- Default model: `openai/gpt-4o` **API-key providers** (`setup_api_key_provider`): 1. Check env var → if set, ask to reuse, persist to secrets store diff --git a/src/setup/prompts.rs b/src/setup/prompts.rs index df4cbbc2..a52a8b68 100644 --- a/src/setup/prompts.rs +++ b/src/setup/prompts.rs @@ -200,6 +200,16 @@ fn read_secret_line() -> io::Result { let mut input = String::new(); let mut stdout = io::stdout(); + // Drain any residual key events (e.g. Enter from a prior `read_line` prompt) + // that are already queued before we start reading. Without this, on + // Windows the leftover Enter is immediately consumed and the function + // returns an empty string before the user can type anything. + // Uses Duration::ZERO so we never block waiting for new input — only + // events already in the queue are consumed. + while event::poll(std::time::Duration::ZERO)? { + let _ = event::read()?; + } + loop { if let Event::Key(KeyEvent { code, modifiers, .. diff --git a/src/testing/credentials.rs b/src/testing/credentials.rs new file mode 100644 index 00000000..9492b69b --- /dev/null +++ b/src/testing/credentials.rs @@ -0,0 +1,134 @@ +//! Centralized fake credential constants for tests. +//! +//! All values here are intentionally fake. Centralizing them makes security +//! audits trivial (one file to verify) and eliminates duplication across +//! the test suite. + +use std::sync::Arc; + +use secrecy::SecretString; + +use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + +// ── Encryption keys ────────────────────────────────────────────────────── + +/// 32-character key string for `SecretsCrypto::new()` in tests. +pub const TEST_CRYPTO_KEY: &str = "0123456789abcdef0123456789abcdef"; + +/// 32+ char key for web gateway `SecretsCrypto` in tests. +pub const TEST_GATEWAY_CRYPTO_KEY: &str = "test-key-at-least-32-chars-long!!"; + +// ── OpenAI-style API keys ──────────────────────────────────────────────── + +/// Generic OpenAI-style test API key. +pub const TEST_OPENAI_API_KEY: &str = "sk-test123"; + +/// OpenAI API key with longer format (config round-trip tests). +pub const TEST_OPENAI_API_KEY_LONG: &str = "sk-test-key-1234567890"; + +/// Short OpenAI-style key for secrets store accessibility tests. +pub const TEST_OPENAI_API_KEY_SHORT: &str = "sk-test"; + +/// OpenAI API key used in embeddings config issue-129 test. +pub const TEST_OPENAI_API_KEY_ISSUE_129: &str = "sk-test-key-for-issue-129"; + +// ── Anthropic keys ─────────────────────────────────────────────────────── + +/// Anthropic OAuth token for config tests. +pub const TEST_ANTHROPIC_OAUTH_TOKEN: &str = "sk-ant-oat01-test-token"; + +/// Anthropic API key for priority tests. +pub const TEST_ANTHROPIC_API_KEY: &str = "sk-ant-priority-key"; + +/// Anthropic OAuth token for sandbox config parse tests. +pub const TEST_ANTHROPIC_OAUTH_BASIC: &str = "sk-ant-oat01-basic"; + +/// Anthropic OAuth token in nested JSON parse test. +pub const TEST_ANTHROPIC_OAUTH_NESTED: &str = "sk-ant-oat01-primary-token"; + +// ── Google OAuth ───────────────────────────────────────────────────────── + +/// Google OAuth access token (standard test). +pub const TEST_GOOGLE_OAUTH_TOKEN: &str = "ya29.test-token"; + +/// Google OAuth access token (fresh/non-expired variant). +pub const TEST_GOOGLE_OAUTH_FRESH: &str = "ya29.fresh-token"; + +/// Google OAuth access token (legacy/no-expiry variant). +pub const TEST_GOOGLE_OAUTH_LEGACY: &str = "ya29.legacy-token"; + +// ── GitHub ─────────────────────────────────────────────────────────────── + +/// GitHub personal access token (test). +pub const TEST_GITHUB_TOKEN: &str = "ghp_test123"; + +// ── Telegram ──────────────────────────────────────────────────────────── + +/// Telegram bot token for credential redaction tests. +pub const TEST_TELEGRAM_BOT_TOKEN: &str = "telegram-test-bot-token-not-a-real-token"; + +// ── OAuth client credentials ──────────────────────────────────────────── + +/// OAuth client ID for token refresh tests. +pub const TEST_OAUTH_CLIENT_ID: &str = "test-client-id"; + +/// OAuth client secret for token refresh tests. +pub const TEST_OAUTH_CLIENT_SECRET: &str = "test-client-secret"; + +// ── Bearer/auth tokens ────────────────────────────────────────────────── + +/// Generic test bearer token. +pub const TEST_BEARER_TOKEN: &str = "test-token"; + +/// Bearer token with suffix (wasm wrapper credential injection). +pub const TEST_BEARER_TOKEN_123: &str = "test-token-123"; + +/// Auth token used by web gateway middleware tests. +pub const TEST_AUTH_SECRET_TOKEN: &str = "secret-token"; + +// ── Stripe ────────────────────────────────────────────────────────────── + +/// Stripe-style test key. +pub const TEST_STRIPE_KEY: &str = "sk_test_fake123"; + +// ── Redaction test values ─────────────────────────────────────────────── + +/// Secret-prefixed key for redaction/sanitization tests. +pub const TEST_REDACT_SECRET: &str = "sk-secret"; + +/// Secret-prefixed key with suffix for redaction tests. +pub const TEST_REDACT_SECRET_123: &str = "sk-secret-123"; + +// ── Session tokens ────────────────────────────────────────────────────── + +/// Generic session token for persistence tests. +pub const TEST_SESSION_TOKEN: &str = "test_token_123"; + +/// NEAR AI session token variant A. +pub const TEST_SESSION_NEARAI_ABC: &str = "sess_abc123"; + +/// NEAR AI session token variant B. +pub const TEST_SESSION_NEARAI_XYZ: &str = "sess_xyz789"; + +// ── Generic ────────────────────────────────────────────────────────────── + +/// Generic test API key for LLM config, embedding config, nearai tests. +pub const TEST_API_KEY: &str = "test-key"; + +/// Stored secret value for create-and-get tests. +pub const TEST_SECRET_VALUE: &str = "sk-test-12345"; + +/// HTTP webhook secret for channel tests. +pub const TEST_HTTP_SECRET: &str = "test-secret-123"; + +// ── Helpers ────────────────────────────────────────────────────────────── + +/// Create an `InMemorySecretsStore` backed by [`TEST_CRYPTO_KEY`]. +/// +/// Replaces the duplicated `test_store()` pattern found across multiple +/// test modules. +pub fn test_secrets_store() -> InMemorySecretsStore { + let crypto = + Arc::new(SecretsCrypto::new(SecretString::from(TEST_CRYPTO_KEY.to_string())).unwrap()); + InMemorySecretsStore::new(crypto) +} diff --git a/src/testing.rs b/src/testing/mod.rs similarity index 99% rename from src/testing.rs rename to src/testing/mod.rs index 8f57cffc..97612887 100644 --- a/src/testing.rs +++ b/src/testing/mod.rs @@ -18,6 +18,8 @@ //! } //! ``` +pub mod credentials; + use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; diff --git a/src/tools/builder/core.rs b/src/tools/builder/core.rs index 0400d24d..190fd21e 100644 --- a/src/tools/builder/core.rs +++ b/src/tools/builder/core.rs @@ -509,7 +509,8 @@ Create alongside the .wasm file to grant capabilities: let mut iteration = 0; // Create reasoning engine - let reasoning = Reasoning::new(self.llm.clone()); + let reasoning = + Reasoning::new(self.llm.clone()).with_model_name(self.llm.active_model_name()); // Build initial context let tool_defs = self.get_build_tools().await; @@ -810,7 +811,8 @@ Create alongside the .wasm file to grant capabilities: impl SoftwareBuilder for LlmSoftwareBuilder { async fn analyze(&self, description: &str) -> Result { // Use LLM to parse the description - let reasoning = Reasoning::new(self.llm.clone()); + let reasoning = + Reasoning::new(self.llm.clone()).with_model_name(self.llm.active_model_name()); let prompt = format!( r#"Analyze this software requirement and extract structured information. diff --git a/src/tools/builtin/extension_tools.rs b/src/tools/builtin/extension_tools.rs index ce8a06a8..793ae610 100644 --- a/src/tools/builtin/extension_tools.rs +++ b/src/tools/builtin/extension_tools.rs @@ -768,11 +768,11 @@ mod tests { /// Create a stub manager for schema tests (these don't call execute). fn test_manager_stub() -> Arc { use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + use crate::testing::credentials::TEST_CRYPTO_KEY; use crate::tools::ToolRegistry; use crate::tools::mcp::session::McpSessionManager; - let master_key = - secrecy::SecretString::from("0123456789abcdef0123456789abcdef".to_string()); + let master_key = secrecy::SecretString::from(TEST_CRYPTO_KEY.to_string()); let crypto = Arc::new(SecretsCrypto::new(master_key).unwrap()); Arc::new(ExtensionManager::new( diff --git a/src/tools/builtin/http.rs b/src/tools/builtin/http.rs index c6e09139..3b506c24 100644 --- a/src/tools/builtin/http.rs +++ b/src/tools/builtin/http.rs @@ -609,6 +609,7 @@ impl Tool for HttpTool { #[cfg(test)] mod tests { use super::*; + use crate::testing::credentials::{TEST_OPENAI_API_KEY, test_secrets_store}; #[test] fn test_http_tool_schema_headers_is_array() { @@ -868,12 +869,7 @@ mod tests { let tool = HttpTool::new().with_credentials( registry, // secrets_store is not used in requires_approval, just needs to be present - Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( - crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "0123456789abcdef0123456789abcdef".to_string(), - )) - .unwrap(), - ))), + Arc::new(test_secrets_store()), ); let params = serde_json::json!({ @@ -890,15 +886,7 @@ mod tests { let registry = Arc::new(SharedCredentialRegistry::new()); // Empty registry - no credential mappings - let tool = HttpTool::new().with_credentials( - registry, - Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( - crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "0123456789abcdef0123456789abcdef".to_string(), - )) - .unwrap(), - ))), - ); + let tool = HttpTool::new().with_credentials(registry, Arc::new(test_secrets_store())); let params = serde_json::json!({ "method": "GET", @@ -926,7 +914,7 @@ mod tests { let params = serde_json::json!({ "method": "GET", "url": "https://example.com", - "headers": {"X-Custom": "Bearer sk-test123"} + "headers": {"X-Custom": format!("Bearer {TEST_OPENAI_API_KEY}")} }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } @@ -957,15 +945,7 @@ mod tests { let registry = Arc::new(SharedCredentialRegistry::new()); registry.add_mappings(vec![CredentialMapping::bearer("test_key", "api.test.com")]); - let tool = HttpTool::new().with_credentials( - registry, - Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( - crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( - "0123456789abcdef0123456789abcdef".to_string(), - )) - .unwrap(), - ))), - ); + let tool = HttpTool::new().with_credentials(registry, Arc::new(test_secrets_store())); // These calls should not panic in multi-thread runtime let params_no_auth = serde_json::json!({ diff --git a/src/tools/builtin/job.rs b/src/tools/builtin/job.rs index f502259f..880f8622 100644 --- a/src/tools/builtin/job.rs +++ b/src/tools/builtin/job.rs @@ -1748,14 +1748,10 @@ mod tests { #[tokio::test] async fn test_parse_credentials_missing_secret() { - use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; - use secrecy::SecretString; + use crate::testing::credentials::test_secrets_store; let manager = Arc::new(ContextManager::new(5)); - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let secrets: Arc = - Arc::new(InMemorySecretsStore::new(crypto)); + let secrets: Arc = Arc::new(test_secrets_store()); let tool = CreateJobTool::new(manager).with_secrets(Arc::clone(&secrets)); @@ -1772,20 +1768,17 @@ mod tests { #[tokio::test] async fn test_parse_credentials_valid() { - use crate::secrets::{CreateSecretParams, InMemorySecretsStore, SecretsCrypto}; - use secrecy::SecretString; + use crate::secrets::CreateSecretParams; + use crate::testing::credentials::{TEST_GITHUB_TOKEN, test_secrets_store}; let manager = Arc::new(ContextManager::new(5)); - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let secrets: Arc = - Arc::new(InMemorySecretsStore::new(Arc::clone(&crypto))); + let secrets: Arc = Arc::new(test_secrets_store()); // Store a secret secrets .create( "user1", - CreateSecretParams::new("github_token", "ghp_test123"), + CreateSecretParams::new("github_token", TEST_GITHUB_TOKEN), ) .await .unwrap(); diff --git a/src/tools/builtin/mod.rs b/src/tools/builtin/mod.rs index 0b181986..b52502c9 100644 --- a/src/tools/builtin/mod.rs +++ b/src/tools/builtin/mod.rs @@ -32,8 +32,8 @@ pub use memory::{MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTo pub use message::MessageTool; pub use restart::RestartTool; pub use routine::{ - RoutineCreateTool, RoutineDeleteTool, RoutineFireTool, RoutineHistoryTool, RoutineListTool, - RoutineUpdateTool, + EventEmitTool, RoutineCreateTool, RoutineDeleteTool, RoutineFireTool, RoutineHistoryTool, + RoutineListTool, RoutineUpdateTool, }; pub use secrets_tools::{SecretDeleteTool, SecretListTool}; pub use shell::ShellTool; diff --git a/src/tools/builtin/routine.rs b/src/tools/builtin/routine.rs index 090d1ff9..573c3c60 100644 --- a/src/tools/builtin/routine.rs +++ b/src/tools/builtin/routine.rs @@ -1,12 +1,13 @@ //! LLM-facing tools for managing routines. //! -//! Six tools let the agent manage routines conversationally: +//! Seven tools let the agent manage routines conversationally: //! - `routine_create` - Create a new routine //! - `routine_list` - List all routines with status //! - `routine_update` - Modify or toggle a routine //! - `routine_delete` - Remove a routine //! - `routine_fire` - Manually trigger a routine //! - `routine_history` - View past runs +//! - `event_emit` - Emit a structured system event to `system_event`-triggered routines use std::sync::Arc; use std::time::Duration; @@ -44,7 +45,7 @@ impl Tool for RoutineCreateTool { fn description(&self) -> &str { "Create a new routine (scheduled or event-driven task). \ - Supports cron schedules, event pattern matching, webhooks, and manual triggers. \ + Supports cron schedules, event pattern matching, system events, and manual triggers. \ Use this when the user wants something to happen periodically or reactively." } @@ -62,7 +63,7 @@ impl Tool for RoutineCreateTool { }, "trigger_type": { "type": "string", - "enum": ["cron", "event", "webhook", "manual"], + "enum": ["cron", "event", "system_event", "manual"], "description": "When the routine fires" }, "schedule": { @@ -77,6 +78,18 @@ impl Tool for RoutineCreateTool { "type": "string", "description": "Optional channel filter for event trigger (e.g. 'telegram')" }, + "event_source": { + "type": "string", + "description": "Event source for system_event triggers (e.g. 'github')" + }, + "event_type": { + "type": "string", + "description": "Event type for system_event triggers (e.g. 'issue.opened')" + }, + "event_filters": { + "type": "object", + "description": "Optional exact-match filters against payload fields for system_event triggers. Values can be strings, numbers, or booleans." + }, "prompt": { "type": "string", "description": "The prompt/instructions for the routine" @@ -190,10 +203,41 @@ impl Tool for RoutineCreateTool { pattern: pattern.to_string(), } } - "webhook" => Trigger::Webhook { - path: None, - secret: None, - }, + "system_event" => { + let source = params + .get("event_source") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + ToolError::InvalidParameters( + "system_event trigger requires 'event_source'".to_string(), + ) + })?; + let event_type = params + .get("event_type") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + ToolError::InvalidParameters( + "system_event trigger requires 'event_type'".to_string(), + ) + })?; + let filters = params + .get("event_filters") + .and_then(|v| v.as_object()) + .map(|obj| { + obj.iter() + .filter_map(|(k, v)| { + crate::agent::routine::json_value_as_filter_string(v) + .map(|s| (k.to_string(), s)) + }) + .collect::>() + }) + .unwrap_or_default(); + Trigger::SystemEvent { + source: source.to_string(), + event_type: event_type.to_string(), + filters, + } + } "manual" => Trigger::Manual, other => { return Err(ToolError::InvalidParameters(format!( @@ -296,7 +340,10 @@ impl Tool for RoutineCreateTool { .map_err(|e| ToolError::ExecutionFailed(format!("failed to create routine: {e}")))?; // Refresh event cache if this is an event trigger - if routine.trigger.type_tag() == "event" { + if matches!( + routine.trigger, + Trigger::Event { .. } | Trigger::SystemEvent { .. } + ) { self.engine.refresh_event_cache().await; } @@ -801,3 +848,87 @@ impl Tool for RoutineHistoryTool { false } } + +// ==================== event_emit ==================== + +pub struct EventEmitTool { + engine: Arc, +} + +impl EventEmitTool { + pub fn new(engine: Arc) -> Self { + Self { engine } + } +} + +#[async_trait] +impl Tool for EventEmitTool { + fn name(&self) -> &str { + "event_emit" + } + + fn description(&self) -> &str { + "Emit a structured system event to routines with a system_event trigger. \ + Use this to trigger routines from tool workflows without waiting for cron." + } + + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + // Emitting an event can fire system_event routines that dispatch full_jobs + // with pre-authorized Always-gated tools — same escalation risk as routine_fire. + ApprovalRequirement::UnlessAutoApproved + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "event_source": { + "type": "string", + "description": "Event source (e.g. 'github', 'workflow', 'tool')" + }, + "event_type": { + "type": "string", + "description": "Event type (e.g. 'issue.opened', 'pr.ready')" + }, + "payload": { + "type": "object", + "description": "Structured event payload" + } + }, + "required": ["event_source", "event_type"] + }) + } + + async fn execute( + &self, + params: serde_json::Value, + ctx: &JobContext, + ) -> Result { + let start = std::time::Instant::now(); + + let source = require_str(¶ms, "event_source")?; + let event_type = require_str(¶ms, "event_type")?; + let payload = params + .get("payload") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + + let fired = self + .engine + .emit_system_event(source, event_type, &payload, Some(&ctx.user_id)) + .await; + + let result = serde_json::json!({ + "event_source": source, + "event_type": event_type, + "user_id": &ctx.user_id, + "fired_routines": fired, + }); + + Ok(ToolOutput::success(result, start.elapsed())) + } + + fn requires_sanitization(&self) -> bool { + true + } +} diff --git a/src/tools/builtin/secrets_tools.rs b/src/tools/builtin/secrets_tools.rs index 8d5c8d62..af2d035b 100644 --- a/src/tools/builtin/secrets_tools.rs +++ b/src/tools/builtin/secrets_tools.rs @@ -158,16 +158,13 @@ impl Tool for SecretDeleteTool { mod tests { use std::sync::Arc; - use secrecy::SecretString; - use super::*; use crate::context::JobContext; - use crate::secrets::{CreateSecretParams, InMemorySecretsStore, SecretsCrypto}; + use crate::secrets::CreateSecretParams; + use crate::testing::credentials::{TEST_OPENAI_API_KEY_SHORT, test_secrets_store}; - fn test_store() -> Arc { - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - Arc::new(InMemorySecretsStore::new(crypto)) + fn test_store() -> Arc { + Arc::new(test_secrets_store()) } fn test_ctx() -> JobContext { @@ -183,7 +180,7 @@ mod tests { store .create( &ctx.user_id, - CreateSecretParams::new("openai_key", "sk-test"), + CreateSecretParams::new("openai_key", TEST_OPENAI_API_KEY_SHORT), ) .await .unwrap(); diff --git a/src/tools/execute.rs b/src/tools/execute.rs new file mode 100644 index 00000000..7c82d7ff --- /dev/null +++ b/src/tools/execute.rs @@ -0,0 +1,391 @@ +//! Shared tool execution pipeline. +//! +//! Provides a single implementation of the validate → timeout → execute → serialize +//! pipeline used by all agentic loop consumers (chat, job, container) and the +//! scheduler's subtask execution. + +use crate::context::JobContext; +use crate::error::Error; +use crate::llm::ChatMessage; +use crate::safety::SafetyLayer; +use crate::tools::{ToolRegistry, redact_params}; + +/// Execute a tool with safety checks: lookup → validate → timeout → execute → serialize. +/// +/// This is the single canonical implementation of tool execution. All consumers +/// (chat dispatcher, job worker, container runtime, scheduler subtasks) use this +/// function instead of maintaining their own copies. +pub async fn execute_tool_with_safety( + tools: &ToolRegistry, + safety: &SafetyLayer, + tool_name: &str, + params: &serde_json::Value, + job_ctx: &JobContext, +) -> Result { + let tool = tools + .get(tool_name) + .await + .ok_or_else(|| crate::error::ToolError::NotFound { + name: tool_name.to_string(), + })?; + + // Validate tool parameters + let validation = safety.validator().validate_tool_params(params); + if !validation.is_valid { + let details = validation + .errors + .iter() + .map(|e| format!("{}: {}", e.field, e.message)) + .collect::>() + .join("; "); + return Err(crate::error::ToolError::InvalidParameters { + name: tool_name.to_string(), + reason: format!("Invalid tool parameters: {}", details), + } + .into()); + } + + let safe_params = redact_params(params, tool.sensitive_params()); + tracing::debug!( + tool = %tool_name, + params = %safe_params, + "Tool call started" + ); + + // Execute with per-tool timeout + let timeout = tool.execution_timeout(); + let start = std::time::Instant::now(); + let result = tokio::time::timeout(timeout, async { + tool.execute(params.clone(), job_ctx).await + }) + .await; + let elapsed = start.elapsed(); + + match &result { + Ok(Ok(output)) => { + let result_size = serde_json::to_string(&output.result) + .map(|s| s.len()) + .unwrap_or(0); + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + result_size_bytes = result_size, + "Tool call succeeded" + ); + } + Ok(Err(e)) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + error = %e, + "Tool call failed" + ); + } + Err(_) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + timeout_secs = timeout.as_secs(), + "Tool call timed out" + ); + } + } + + let result = result + .map_err(|_| crate::error::ToolError::Timeout { + name: tool_name.to_string(), + timeout, + })? + .map_err(|e| crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: e.to_string(), + })?; + + serde_json::to_string_pretty(&result.result).map_err(|e| { + crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: format!("Failed to serialize result: {}", e), + } + .into() + }) +} + +/// Process a tool result into a `ChatMessage::tool_result` with safety sanitization. +/// +/// On success: sanitize → wrap → ChatMessage::tool_result. +/// On error: format error → ChatMessage::tool_result. +/// +/// Returns the content string and the ChatMessage. +pub fn process_tool_result( + safety: &SafetyLayer, + tool_name: &str, + tool_call_id: &str, + result: &Result, +) -> (String, ChatMessage) { + let content = match result { + Ok(output) => { + let sanitized = safety.sanitize_tool_output(tool_name, output); + safety.wrap_for_llm(tool_name, &sanitized.content, sanitized.was_modified) + } + Err(e) => format!("Error: {}", e), + }; + let message = ChatMessage::tool_result(tool_call_id, tool_name, content.clone()); + (content, message) +} + +/// Execute a tool with safety checks, returning a string error (for container runtime). +/// +/// This is a thin wrapper around `execute_tool_with_safety` that converts +/// `Error` to `String` for the container runtime's simpler error model. +pub async fn execute_tool_simple( + tools: &ToolRegistry, + safety: &SafetyLayer, + tool_name: &str, + params: &serde_json::Value, + job_ctx: &JobContext, +) -> Result { + execute_tool_with_safety(tools, safety, tool_name, params, job_ctx) + .await + .map_err(|e| e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tools::tool::{Tool, ToolError, ToolOutput}; + use std::sync::Arc; + use std::time::Duration; + + struct EchoTool; + + #[async_trait::async_trait] + impl Tool for EchoTool { + fn name(&self) -> &str { + "echo" + } + fn description(&self) -> &str { + "Echoes input" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + params: serde_json::Value, + _ctx: &JobContext, + ) -> Result { + Ok(ToolOutput::success(params, Duration::default())) + } + fn requires_sanitization(&self) -> bool { + false + } + } + + struct FailTool; + + #[async_trait::async_trait] + impl Tool for FailTool { + fn name(&self) -> &str { + "fail_tool" + } + fn description(&self) -> &str { + "Always fails" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _: serde_json::Value, + _: &JobContext, + ) -> Result { + Err(ToolError::ExecutionFailed( + "intentional failure".to_string(), + )) + } + fn requires_sanitization(&self) -> bool { + false + } + } + + struct SlowTool; + + #[async_trait::async_trait] + impl Tool for SlowTool { + fn name(&self) -> &str { + "slow_tool" + } + fn description(&self) -> &str { + "Sleeps forever" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _: serde_json::Value, + _: &JobContext, + ) -> Result { + tokio::time::sleep(Duration::from_secs(60)).await; + unreachable!() + } + fn execution_timeout(&self) -> Duration { + Duration::from_millis(50) + } + fn requires_sanitization(&self) -> bool { + false + } + } + + fn test_safety() -> SafetyLayer { + SafetyLayer::new(&crate::config::SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + }) + } + + fn test_job_ctx() -> JobContext { + JobContext::default() + } + + async fn registry_with(tools: Vec>) -> ToolRegistry { + let registry = ToolRegistry::new(); + for tool in tools { + registry.register(tool).await; + } + registry + } + + #[tokio::test] + async fn test_execute_success() { + let registry = registry_with(vec![Arc::new(EchoTool)]).await; + let safety = test_safety(); + let params = serde_json::json!({"message": "hello"}); + + let result = + execute_tool_with_safety(®istry, &safety, "echo", ¶ms, &test_job_ctx()).await; + + assert!(result.is_ok(), "Echo tool should succeed"); + let output = result.unwrap(); + assert!( + output.contains("hello"), + "Output should contain the echoed input" + ); + } + + #[tokio::test] + async fn test_execute_missing_tool() { + let registry = registry_with(vec![]).await; + let safety = test_safety(); + + let result = execute_tool_with_safety( + ®istry, + &safety, + "nonexistent", + &serde_json::json!({}), + &test_job_ctx(), + ) + .await; + + assert!(result.is_err(), "Missing tool should return error"); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("nonexistent") || err.contains("not found"), + "Error should mention the tool: {}", + err + ); + } + + #[tokio::test] + async fn test_execute_tool_failure() { + let registry = registry_with(vec![Arc::new(FailTool)]).await; + let safety = test_safety(); + + let result = execute_tool_with_safety( + ®istry, + &safety, + "fail_tool", + &serde_json::json!({}), + &test_job_ctx(), + ) + .await; + + assert!(result.is_err(), "FailTool should return error"); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("intentional failure"), + "Error should contain the failure reason: {}", + err + ); + } + + #[tokio::test] + async fn test_execute_tool_timeout() { + let registry = registry_with(vec![Arc::new(SlowTool)]).await; + let safety = test_safety(); + + let start = std::time::Instant::now(); + let result = execute_tool_with_safety( + ®istry, + &safety, + "slow_tool", + &serde_json::json!({}), + &test_job_ctx(), + ) + .await; + let elapsed = start.elapsed(); + + assert!(result.is_err(), "SlowTool should timeout"); + let err = result.unwrap_err().to_string(); + assert!( + err.to_lowercase().contains("timeout") || err.to_lowercase().contains("timed out"), + "Error should mention timeout: {}", + err + ); + assert!( + elapsed < Duration::from_secs(1), + "Should timeout quickly, not wait 60s" + ); + } + + #[test] + fn test_process_tool_result_success() { + let safety = test_safety(); + let result: Result = Ok("tool output data".to_string()); + + let (content, message) = process_tool_result(&safety, "echo", "call_1", &result); + + assert!( + content.contains("tool_output"), + "Content should be XML-wrapped: {}", + content + ); + assert!( + content.contains("tool output data"), + "Content should contain the output: {}", + content + ); + assert_eq!(message.role, crate::llm::Role::Tool); + assert_eq!(message.name.as_deref(), Some("echo")); + } + + #[test] + fn test_process_tool_result_error() { + let safety = test_safety(); + let result: Result = Err("something went wrong".to_string()); + + let (content, message) = process_tool_result(&safety, "echo", "call_1", &result); + + assert!( + content.contains("Error:"), + "Error content should start with 'Error:': {}", + content + ); + assert!( + content.contains("something went wrong"), + "Error content should contain the message: {}", + content + ); + assert_eq!(message.role, crate::llm::Role::Tool); + } +} diff --git a/src/tools/mcp/client.rs b/src/tools/mcp/client.rs index cd74d572..61c9d5c7 100644 --- a/src/tools/mcp/client.rs +++ b/src/tools/mcp/client.rs @@ -490,6 +490,12 @@ impl Tool for McpToolWrapper { _ctx: &JobContext, ) -> Result { let start = std::time::Instant::now(); + + // Strip top-level null values before forwarding — LLMs often emit + // `"field": null` for optional params, but many MCP servers reject + // explicit nulls for fields that should simply be absent. + let params = strip_top_level_nulls(params); + let result = self.client.call_tool(&self.tool.name, params).await?; let content: String = result .content @@ -516,9 +522,22 @@ impl Tool for McpToolWrapper { } } -/// Sanitize an HTTP error response body for safe display. +/// Remove top-level keys whose value is JSON null from an object. /// -/// Detects full HTML error pages (containing ` serde_json::Value { + match value { + serde_json::Value::Object(map) => { + let filtered = map.into_iter().filter(|(_, v)| !v.is_null()).collect(); + serde_json::Value::Object(filtered) + } + other => other, + } +} + #[cfg(test)] mod tests { use super::*; @@ -806,4 +825,40 @@ mod tests { let mock_non_http = MockTransport::new(false, vec![]); assert!(!mock_non_http.supports_http_features()); } + + #[test] + fn test_strip_top_level_nulls_removes_null_fields() { + let input = serde_json::json!({ + "query": "search term", + "sort": null, + "filter": null, + "page_size": 10 + }); + let result = strip_top_level_nulls(input); + let obj = result.as_object().unwrap(); + assert_eq!(obj.len(), 2); + assert_eq!(obj["query"], "search term"); + assert_eq!(obj["page_size"], 10); + assert!(!obj.contains_key("sort")); + assert!(!obj.contains_key("filter")); + } + + #[test] + fn test_strip_top_level_nulls_preserves_non_objects() { + let input = serde_json::json!("just a string"); + let result = strip_top_level_nulls(input.clone()); + assert_eq!(result, input); + } + + #[test] + fn test_strip_top_level_nulls_preserves_nested_nulls() { + let input = serde_json::json!({ + "outer": { "inner": null }, + "top_null": null + }); + let result = strip_top_level_nulls(input); + let obj = result.as_object().unwrap(); + assert_eq!(obj.len(), 1); + assert!(obj["outer"]["inner"].is_null()); + } } diff --git a/src/tools/mod.rs b/src/tools/mod.rs index d379d474..833d278b 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -9,6 +9,7 @@ pub mod builder; pub mod builtin; +pub mod execute; pub mod mcp; pub mod rate_limiter; pub mod schema_validator; diff --git a/src/tools/registry.rs b/src/tools/registry.rs index c6612b32..7054eea3 100644 --- a/src/tools/registry.rs +++ b/src/tools/registry.rs @@ -64,6 +64,7 @@ const PROTECTED_TOOL_NAMES: &[&str] = &[ "routine_delete", "routine_fire", "routine_history", + "event_emit", "skill_list", "skill_search", "skill_install", @@ -136,7 +137,7 @@ impl ToolRegistry { return; } self.tools.write().await.insert(name.clone(), tool); - tracing::debug!("Registered tool: {}", name); + tracing::trace!("Registered tool: {}", name); } /// Register a tool (sync version for startup, marks as built-in). @@ -427,8 +428,8 @@ impl ToolRegistry { engine: Arc, ) { use crate::tools::builtin::{ - RoutineCreateTool, RoutineDeleteTool, RoutineFireTool, RoutineHistoryTool, - RoutineListTool, RoutineUpdateTool, + EventEmitTool, RoutineCreateTool, RoutineDeleteTool, RoutineFireTool, + RoutineHistoryTool, RoutineListTool, RoutineUpdateTool, }; self.register_sync(Arc::new(RoutineCreateTool::new( Arc::clone(&store), @@ -448,7 +449,8 @@ impl ToolRegistry { Arc::clone(&engine), ))); self.register_sync(Arc::new(RoutineHistoryTool::new(store))); - tracing::debug!("Registered 6 routine management tools"); + self.register_sync(Arc::new(EventEmitTool::new(engine))); + tracing::debug!("Registered 7 routine management tools"); } /// Register message tool for sending messages to channels. diff --git a/src/tools/schema_validator.rs b/src/tools/schema_validator.rs index 8da0b613..a5b8fd40 100644 --- a/src/tools/schema_validator.rs +++ b/src/tools/schema_validator.rs @@ -565,12 +565,19 @@ mod tests { "description": { "type": "string", "description": "What it does" }, "trigger_type": { "type": "string", - "enum": ["cron", "event", "webhook", "manual"], + "enum": ["cron", "event", "system_event", "manual"], "description": "When the routine fires" }, "schedule": { "type": "string", "description": "Cron expression" }, "event_pattern": { "type": "string", "description": "Regex pattern" }, "event_channel": { "type": "string", "description": "Channel filter" }, + "event_source": { "type": "string", "description": "System event source" }, + "event_type": { "type": "string", "description": "System event type" }, + "event_filters": { + "type": "object", + "additionalProperties": { "type": "string" }, + "description": "Exact-match payload filters" + }, "prompt": { "type": "string", "description": "Instructions" }, "context_paths": { "type": "array", @@ -647,6 +654,18 @@ mod tests { "required": ["name"] }), ), + ( + "event_emit", + serde_json::json!({ + "type": "object", + "properties": { + "event_source": { "type": "string", "description": "Event source" }, + "event_type": { "type": "string", "description": "Event type" }, + "payload": { "type": "object", "description": "Event payload", "properties": {} } + }, + "required": ["event_source", "event_type"] + }), + ), // Job tools with complex deps ( "job_events", diff --git a/src/tools/tool.rs b/src/tools/tool.rs index 2e1b5183..8bf29168 100644 --- a/src/tools/tool.rs +++ b/src/tools/tool.rs @@ -480,6 +480,7 @@ pub fn validate_tool_schema(schema: &serde_json::Value, path: &str) -> Vec String { #[cfg(test)] mod tests { use std::collections::HashMap; - use std::sync::Arc; - - use secrecy::SecretString; use crate::secrets::{ CreateSecretParams, CredentialLocation, CredentialMapping, InMemorySecretsStore, - SecretsCrypto, SecretsStore, + SecretsStore, }; + use crate::testing::credentials::{TEST_OPENAI_API_KEY, test_secrets_store}; use crate::tools::wasm::credential_injector::{ CredentialInjector, base64_encode, host_matches_pattern, }; fn test_store() -> InMemorySecretsStore { - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - InMemorySecretsStore::new(crypto) + test_secrets_store() } #[test] @@ -406,7 +402,10 @@ mod tests { async fn test_inject_bearer() { let store = test_store(); store - .create("user1", CreateSecretParams::new("openai_key", "sk-test123")) + .create( + "user1", + CreateSecretParams::new("openai_key", TEST_OPENAI_API_KEY), + ) .await .unwrap(); @@ -428,7 +427,7 @@ mod tests { assert_eq!( result.headers.get("Authorization"), - Some(&"Bearer sk-test123".to_string()) + Some(&format!("Bearer {TEST_OPENAI_API_KEY}")) ); } diff --git a/src/tools/wasm/loader.rs b/src/tools/wasm/loader.rs index 07319f21..afa471a1 100644 --- a/src/tools/wasm/loader.rs +++ b/src/tools/wasm/loader.rs @@ -694,6 +694,7 @@ mod tests { use tempfile::TempDir; + use crate::testing::credentials::{TEST_OAUTH_CLIENT_ID, TEST_OAUTH_CLIENT_SECRET}; use crate::tools::wasm::loader::{WasmLoadError, check_wit_version_compat, discover_tools}; #[test] @@ -834,8 +835,8 @@ mod tests { oauth: Some(OAuthConfigSchema { authorization_url: "https://accounts.google.com/o/oauth2/v2/auth".to_string(), token_url: "https://oauth2.googleapis.com/token".to_string(), - client_id: Some("test-client-id".to_string()), - client_secret: Some("test-client-secret".to_string()), + client_id: Some(TEST_OAUTH_CLIENT_ID.to_string()), + client_secret: Some(TEST_OAUTH_CLIENT_SECRET.to_string()), ..Default::default() }), ..Default::default() @@ -848,8 +849,11 @@ mod tests { let config = config.unwrap(); assert_eq!(config.token_url, "https://oauth2.googleapis.com/token"); - assert_eq!(config.client_id, "test-client-id"); - assert_eq!(config.client_secret, Some("test-client-secret".to_string())); + assert_eq!(config.client_id, TEST_OAUTH_CLIENT_ID); + assert_eq!( + config.client_secret, + Some(TEST_OAUTH_CLIENT_SECRET.to_string()) + ); assert_eq!(config.secret_name, "google_oauth_token"); assert_eq!(config.provider, Some("google".to_string())); } diff --git a/src/tools/wasm/wrapper.rs b/src/tools/wasm/wrapper.rs index 0bdf8bfa..26c2d5d1 100644 --- a/src/tools/wasm/wrapper.rs +++ b/src/tools/wasm/wrapper.rs @@ -279,6 +279,27 @@ impl near::agent::host::Host for StoreData { let raw_headers: HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); + // Leak scan runs on WASM-provided values BEFORE host credential injection. + // This prevents false positives where the host-injected Bearer token + // (e.g., xoxb- Slack token) triggers the leak detector — WASM never saw + // the real value, so scanning the pre-injection state is correct. + // Inline the scan to avoid allocating a Vec of cloned headers. + let leak_detector = LeakDetector::new(); + leak_detector + .scan_and_clean(&injected_url) + .map_err(|e| format!("Potential secret leak in URL blocked: {}", e))?; + for (name, value) in &raw_headers { + leak_detector.scan_and_clean(value).map_err(|e| { + format!("Potential secret leak in header '{}' blocked: {}", name, e) + })?; + } + if let Some(body_bytes) = body.as_deref() { + let body_str = String::from_utf8_lossy(body_bytes); + leak_detector + .scan_and_clean(&body_str) + .map_err(|e| format!("Potential secret leak in body blocked: {}", e))?; + } + let mut headers: HashMap = raw_headers .into_iter() .map(|(k, v)| { @@ -297,16 +318,6 @@ impl near::agent::host::Host for StoreData { self.inject_host_credentials(&host, &mut headers, &mut url); } - let leak_detector = LeakDetector::new(); - let header_vec: Vec<(String, String)> = headers - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); - - leak_detector - .scan_http_request(&url, &header_vec, body.as_deref()) - .map_err(|e| format!("Potential secret leak blocked: {}", e))?; - // Get the max response size from capabilities (default 10MB). let max_response_bytes = self .host_state @@ -1212,6 +1223,11 @@ fn coerce_params_to_schema( mod tests { use std::sync::Arc; + use crate::testing::credentials::{ + TEST_BEARER_TOKEN_123, TEST_GOOGLE_OAUTH_FRESH, TEST_GOOGLE_OAUTH_LEGACY, + TEST_GOOGLE_OAUTH_TOKEN, TEST_OAUTH_CLIENT_ID, TEST_OAUTH_CLIENT_SECRET, + test_secrets_store, + }; use crate::tools::wasm::capabilities::Capabilities; use crate::tools::wasm::runtime::{WasmRuntimeConfig, WasmToolRuntime}; @@ -1279,12 +1295,12 @@ mod tests { let mut h = HashMap::new(); h.insert( "Authorization".to_string(), - "Bearer test-token-123".to_string(), + format!("Bearer {TEST_BEARER_TOKEN_123}"), ); h }, query_params: HashMap::new(), - secret_value: "test-token-123".to_string(), + secret_value: TEST_BEARER_TOKEN_123.to_string(), }]; let store_data = StoreData::new( @@ -1300,7 +1316,7 @@ mod tests { store_data.inject_host_credentials("www.googleapis.com", &mut headers, &mut url); assert_eq!( headers.get("Authorization"), - Some(&"Bearer test-token-123".to_string()) + Some(&format!("Bearer {TEST_BEARER_TOKEN_123}")) ); // Should not inject for non-matching host @@ -1376,13 +1392,9 @@ mod tests { #[tokio::test] async fn test_resolve_host_credentials_no_http_cap() { - use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; use crate::tools::wasm::wrapper::resolve_host_credentials; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); let caps = Capabilities::default(); let result = resolve_host_credentials(&caps, Some(&store), "user1", None).await; @@ -1394,21 +1406,17 @@ mod tests { use std::collections::HashMap; use crate::secrets::{ - CreateSecretParams, CredentialLocation, CredentialMapping, InMemorySecretsStore, - SecretsCrypto, SecretsStore, + CreateSecretParams, CredentialLocation, CredentialMapping, SecretsStore, }; use crate::tools::wasm::capabilities::HttpCapability; use crate::tools::wasm::wrapper::resolve_host_credentials; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); store .create( "user1", - CreateSecretParams::new("google_oauth_token", "ya29.test-token"), + CreateSecretParams::new("google_oauth_token", TEST_GOOGLE_OAUTH_TOKEN), ) .await .unwrap(); @@ -1436,7 +1444,7 @@ mod tests { assert_eq!(result[0].host_patterns, vec!["www.googleapis.com"]); assert_eq!( result[0].headers.get("Authorization"), - Some(&"Bearer ya29.test-token".to_string()) + Some(&format!("Bearer {TEST_GOOGLE_OAUTH_TOKEN}")) ); } @@ -1444,16 +1452,11 @@ mod tests { async fn test_resolve_host_credentials_missing_secret() { use std::collections::HashMap; - use crate::secrets::{ - CredentialLocation, CredentialMapping, InMemorySecretsStore, SecretsCrypto, - }; + use crate::secrets::{CredentialLocation, CredentialMapping}; use crate::tools::wasm::capabilities::HttpCapability; use crate::tools::wasm::wrapper::resolve_host_credentials; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); // No secret stored, should silently skip let mut credentials = HashMap::new(); @@ -1483,23 +1486,19 @@ mod tests { use std::collections::HashMap; use crate::secrets::{ - CreateSecretParams, CredentialLocation, CredentialMapping, InMemorySecretsStore, - SecretsCrypto, SecretsStore, + CreateSecretParams, CredentialLocation, CredentialMapping, SecretsStore, }; use crate::tools::wasm::capabilities::HttpCapability; use crate::tools::wasm::wrapper::{OAuthRefreshConfig, resolve_host_credentials}; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); // Store a token that expires 2 hours from now (well within buffer) let expires_at = chrono::Utc::now() + chrono::Duration::hours(2); store .create( "user1", - CreateSecretParams::new("google_oauth_token", "ya29.fresh-token") + CreateSecretParams::new("google_oauth_token", TEST_GOOGLE_OAUTH_FRESH) .with_expiry(expires_at), ) .await @@ -1525,8 +1524,8 @@ mod tests { let oauth_config = OAuthRefreshConfig { token_url: "https://oauth2.googleapis.com/token".to_string(), - client_id: "test-client-id".to_string(), - client_secret: Some("test-client-secret".to_string()), + client_id: TEST_OAUTH_CLIENT_ID.to_string(), + client_secret: Some(TEST_OAUTH_CLIENT_SECRET.to_string()), secret_name: "google_oauth_token".to_string(), provider: Some("google".to_string()), }; @@ -1537,7 +1536,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!( result[0].headers.get("Authorization"), - Some(&"Bearer ya29.fresh-token".to_string()) + Some(&format!("Bearer {TEST_GOOGLE_OAUTH_FRESH}")) ); } @@ -1546,16 +1545,12 @@ mod tests { use std::collections::HashMap; use crate::secrets::{ - CreateSecretParams, CredentialLocation, CredentialMapping, InMemorySecretsStore, - SecretsCrypto, SecretsStore, + CreateSecretParams, CredentialLocation, CredentialMapping, SecretsStore, }; use crate::tools::wasm::capabilities::HttpCapability; use crate::tools::wasm::wrapper::resolve_host_credentials; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); // Store an expired token let expires_at = chrono::Utc::now() - chrono::Duration::hours(1); @@ -1595,22 +1590,18 @@ mod tests { use std::collections::HashMap; use crate::secrets::{ - CreateSecretParams, CredentialLocation, CredentialMapping, InMemorySecretsStore, - SecretsCrypto, SecretsStore, + CreateSecretParams, CredentialLocation, CredentialMapping, SecretsStore, }; use crate::tools::wasm::capabilities::HttpCapability; use crate::tools::wasm::wrapper::{OAuthRefreshConfig, resolve_host_credentials}; - use secrecy::SecretString; - let key = "0123456789abcdef0123456789abcdef"; - let crypto = Arc::new(SecretsCrypto::new(SecretString::from(key.to_string())).unwrap()); - let store = InMemorySecretsStore::new(crypto); + let store = test_secrets_store(); // Legacy token: no expires_at set store .create( "user1", - CreateSecretParams::new("google_oauth_token", "ya29.legacy-token"), + CreateSecretParams::new("google_oauth_token", TEST_GOOGLE_OAUTH_LEGACY), ) .await .unwrap(); @@ -1635,8 +1626,8 @@ mod tests { let oauth_config = OAuthRefreshConfig { token_url: "https://oauth2.googleapis.com/token".to_string(), - client_id: "test-client-id".to_string(), - client_secret: Some("test-client-secret".to_string()), + client_id: TEST_OAUTH_CLIENT_ID.to_string(), + client_secret: Some(TEST_OAUTH_CLIENT_SECRET.to_string()), secret_name: "google_oauth_token".to_string(), provider: Some("google".to_string()), }; @@ -1647,7 +1638,7 @@ mod tests { assert_eq!(result.len(), 1); assert_eq!( result[0].headers.get("Authorization"), - Some(&"Bearer ya29.legacy-token".to_string()) + Some(&format!("Bearer {TEST_GOOGLE_OAUTH_LEGACY}")) ); } @@ -1792,4 +1783,57 @@ mod tests { // Should remain as string since it can't be parsed assert_eq!(result["count"], serde_json::json!("not-a-number")); } + + /// Regression test: leak scan must run on raw headers (before credential + /// injection), not after. If it ran post-injection, the host-injected + /// Slack bot token (`xoxb-...`) would trigger a Block and reject the + /// tool's own legitimate outbound request. + #[test] + fn test_leak_scan_runs_before_credential_injection() { + use crate::safety::LeakDetector; + + // Simulate pre-injection headers: WASM only sees the placeholder, not the real token. + let raw_headers: Vec<(String, String)> = vec![ + ( + "Authorization".to_string(), + "Bearer {SLACK_BOT_TOKEN}".to_string(), + ), + ("Content-Type".to_string(), "application/json".to_string()), + ]; + + let detector = LeakDetector::new(); + + // Pre-injection scan should pass — placeholders are not secrets. + let pre_result = detector.scan_http_request( + "https://slack.com/api/chat.postMessage", + &raw_headers, + None, + ); + assert!( + pre_result.is_ok(), + "Leak scan on pre-injection headers should pass, but got: {:?}", + pre_result + ); + + // Post-injection headers would contain a real Slack token. + let post_injection_headers: Vec<(String, String)> = vec![ + ( + "Authorization".to_string(), + "Bearer xoxb-1234567890-abcdefghij".to_string(), + ), + ("Content-Type".to_string(), "application/json".to_string()), + ]; + + // Post-injection scan WOULD block — this is the false positive + // that the pre-injection ordering prevents. + let post_result = detector.scan_http_request( + "https://slack.com/api/chat.postMessage", + &post_injection_headers, + None, + ); + assert!( + post_result.is_err(), + "Leak scan on post-injection headers should block the Slack token" + ); + } } diff --git a/src/tunnel/mod.rs b/src/tunnel/mod.rs index 38ad814b..e6245b9e 100644 --- a/src/tunnel/mod.rs +++ b/src/tunnel/mod.rs @@ -294,10 +294,11 @@ mod tests { #[test] fn factory_cloudflare_with_config_ok() { + use crate::testing::credentials::TEST_BEARER_TOKEN; let cfg = TunnelProviderConfig { provider: "cloudflare".into(), cloudflare: Some(CloudflareTunnelConfig { - token: "test-token".into(), + token: TEST_BEARER_TOKEN.into(), }), ..Default::default() }; diff --git a/src/util.rs b/src/util.rs index 0ac7b69d..866f623c 100644 --- a/src/util.rs +++ b/src/util.rs @@ -24,7 +24,7 @@ pub fn floor_char_boundary(s: &str, pos: usize) -> usize { pub fn llm_signals_completion(response: &str) -> bool { let lower = response.to_lowercase(); - // Superset of phrases from agent/worker.rs and worker/runtime.rs. + // Superset of phrases from worker/job.rs and worker/container.rs. let positive_phrases = [ "job is complete", "job is done", diff --git a/src/worker/api.rs b/src/worker/api.rs index d0048afc..459375b4 100644 --- a/src/worker/api.rs +++ b/src/worker/api.rs @@ -419,13 +419,14 @@ fn parse_finish_reason(s: &str) -> FinishReason { #[cfg(test)] mod tests { use super::*; + use crate::testing::credentials::TEST_BEARER_TOKEN; #[test] fn test_url_construction() { let client = WorkerHttpClient::new( "http://host.docker.internal:50051".to_string(), Uuid::nil(), - "test-token".to_string(), + TEST_BEARER_TOKEN.to_string(), ); assert_eq!( @@ -449,7 +450,7 @@ mod tests { let client = WorkerHttpClient::new( "http://host.docker.internal:50051".to_string(), Uuid::nil(), - "test-token".to_string(), + TEST_BEARER_TOKEN.to_string(), ); assert_eq!( diff --git a/src/worker/container.rs b/src/worker/container.rs new file mode 100644 index 00000000..0b7f41d0 --- /dev/null +++ b/src/worker/container.rs @@ -0,0 +1,539 @@ +//! Worker runtime: the main execution loop inside a container. +//! +//! Reuses the existing `Reasoning` and `SafetyLayer` infrastructure but +//! connects to the orchestrator for LLM calls instead of calling APIs directly. +//! Streams real-time events (message, tool_use, tool_result, result) through +//! the orchestrator's job event pipeline for UI visibility. +//! +//! Uses the shared `AgenticLoop` engine via `ContainerDelegate`. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use tokio::sync::Mutex; +use uuid::Uuid; + +use crate::agent::agentic_loop::{ + AgenticLoopConfig, LoopDelegate, LoopOutcome, LoopSignal, TextAction, truncate_for_preview, +}; +use crate::config::SafetyConfig; +use crate::context::JobContext; +use crate::error::WorkerError; +use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext}; +use crate::safety::SafetyLayer; +use crate::tools::ToolRegistry; +use crate::tools::execute::{execute_tool_simple, process_tool_result}; +use crate::worker::api::{CompletionReport, JobEventPayload, StatusUpdate, WorkerHttpClient}; +use crate::worker::proxy_llm::ProxyLlmProvider; + +/// Configuration for the worker runtime. +pub struct WorkerConfig { + pub job_id: Uuid, + pub orchestrator_url: String, + pub max_iterations: u32, + pub timeout: Duration, +} + +impl Default for WorkerConfig { + fn default() -> Self { + Self { + job_id: Uuid::nil(), + orchestrator_url: String::new(), + max_iterations: 50, + timeout: Duration::from_secs(600), + } + } +} + +/// The worker runtime runs inside a Docker container. +/// +/// It connects to the orchestrator over HTTP, fetches its job description, +/// then runs a tool execution loop until the job is complete. Events are +/// streamed to the orchestrator so the UI can show real-time progress. +pub struct WorkerRuntime { + config: WorkerConfig, + client: Arc, + llm: Arc, + safety: Arc, + tools: Arc, + /// Credentials fetched from the orchestrator, injected into child processes + /// via `Command::envs()` rather than mutating the global process environment. + /// + /// Wrapped in `Arc` to avoid deep-cloning the map on every tool invocation. + extra_env: Arc>, +} + +impl WorkerRuntime { + /// Create a new worker runtime. + /// + /// Reads `IRONCLAW_WORKER_TOKEN` from the environment for auth. + pub fn new(config: WorkerConfig) -> Result { + let client = Arc::new(WorkerHttpClient::from_env( + config.orchestrator_url.clone(), + config.job_id, + )?); + + let llm: Arc = Arc::new(ProxyLlmProvider::new( + Arc::clone(&client), + "proxied".to_string(), + )); + + let safety = Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: true, + })); + + let tools = Arc::new(ToolRegistry::new()); + // Register only container-safe tools + tools.register_container_tools(); + + Ok(Self { + config, + client, + llm, + safety, + tools, + extra_env: Arc::new(HashMap::new()), + }) + } + + /// Run the worker until the job is complete or an error occurs. + pub async fn run(mut self) -> Result<(), WorkerError> { + tracing::info!("Worker starting for job {}", self.config.job_id); + + // Fetch job description from orchestrator + let job = self.client.get_job().await?; + + tracing::info!( + "Received job: {} - {}", + job.title, + truncate_for_preview(&job.description, 100) + ); + + // Fetch credentials and store them for injection into child processes + // via Command::envs() (avoids unsafe std::env::set_var in multi-threaded runtime). + let credentials = self.client.fetch_credentials().await?; + { + let mut env_map = HashMap::new(); + for cred in &credentials { + env_map.insert(cred.env_var.clone(), cred.value.clone()); + } + self.extra_env = Arc::new(env_map); + } + if !credentials.is_empty() { + tracing::info!( + "Fetched {} credential(s) for child process injection", + credentials.len() + ); + } + + // Report that we're starting + self.client + .report_status(&StatusUpdate { + state: "in_progress".to_string(), + message: Some("Worker started, beginning execution".to_string()), + iteration: 0, + }) + .await?; + + // Create reasoning engine + let reasoning = Reasoning::new(self.llm.clone()); + + // Build initial context + let mut reason_ctx = ReasoningContext::new().with_job(&job.description); + + reason_ctx.messages.push(ChatMessage::system(format!( + r#"You are an autonomous agent running inside a Docker container. + +Job: {} +Description: {} + +You have tools for shell commands, file operations, and code editing. +Work independently to complete this job. Report when done."#, + job.title, job.description + ))); + + // Load tool definitions + reason_ctx.available_tools = self.tools.tool_definitions().await; + + // Shared iteration tracker — read after the loop to report accurate counts. + let iteration_tracker = Arc::new(Mutex::new(0u32)); + + // Run with timeout using the shared agentic loop + let result = tokio::time::timeout(self.config.timeout, async { + let delegate = ContainerDelegate { + client: self.client.clone(), + safety: self.safety.clone(), + tools: self.tools.clone(), + extra_env: self.extra_env.clone(), + last_output: Mutex::new(String::new()), + iteration_tracker: iteration_tracker.clone(), + }; + + let config = AgenticLoopConfig { + max_iterations: self.config.max_iterations as usize, + enable_tool_intent_nudge: true, + max_tool_intent_nudges: 2, + }; + + crate::agent::agentic_loop::run_agentic_loop( + &delegate, + &reasoning, + &mut reason_ctx, + &config, + ) + .await + }) + .await; + + let iterations = *iteration_tracker.lock().await; + + match result { + Ok(Ok(LoopOutcome::Response(output))) => { + tracing::info!("Worker completed job {} successfully", self.config.job_id); + self.post_event( + "result", + serde_json::json!({ + "success": true, + "message": truncate_for_preview(&output, 2000), + }), + ) + .await; + self.client + .report_complete(&CompletionReport { + success: true, + message: Some(output), + iterations, + }) + .await?; + } + Ok(Ok(LoopOutcome::MaxIterations)) => { + let msg = format!("max iterations ({}) exceeded", self.config.max_iterations); + tracing::warn!("Worker failed for job {}: {}", self.config.job_id, msg); + self.post_event( + "result", + serde_json::json!({ + "success": false, + "message": format!("Execution failed: {}", msg), + }), + ) + .await; + self.client + .report_complete(&CompletionReport { + success: false, + message: Some(format!("Execution failed: {}", msg)), + iterations, + }) + .await?; + } + Ok(Ok(LoopOutcome::Stopped | LoopOutcome::NeedApproval(_))) => { + tracing::info!("Worker for job {} stopped", self.config.job_id); + self.client + .report_complete(&CompletionReport { + success: false, + message: Some("Execution stopped".to_string()), + iterations, + }) + .await?; + } + Ok(Err(e)) => { + tracing::error!("Worker failed for job {}: {}", self.config.job_id, e); + self.post_event( + "result", + serde_json::json!({ + "success": false, + "message": format!("Execution failed: {}", e), + }), + ) + .await; + self.client + .report_complete(&CompletionReport { + success: false, + message: Some(format!("Execution failed: {}", e)), + iterations, + }) + .await?; + } + Err(_) => { + tracing::warn!("Worker timed out for job {}", self.config.job_id); + self.post_event( + "result", + serde_json::json!({ + "success": false, + "message": "Execution timed out", + }), + ) + .await; + self.client + .report_complete(&CompletionReport { + success: false, + message: Some("Execution timed out".to_string()), + iterations, + }) + .await?; + } + } + + Ok(()) + } + + /// Post a job event to the orchestrator (fire-and-forget). + async fn post_event(&self, event_type: &str, data: serde_json::Value) { + self.client + .post_event(&JobEventPayload { + event_type: event_type.to_string(), + data, + }) + .await; + } +} + +/// Container delegate: implements `LoopDelegate` for the Docker container context. +/// +/// Tools execute sequentially. Events are posted to the orchestrator via HTTP. +/// Completion is detected via `llm_signals_completion()`. +struct ContainerDelegate { + client: Arc, + safety: Arc, + tools: Arc, + extra_env: Arc>, + /// Tracks the last successful tool output for the final response. + last_output: Mutex, + /// Tracks the current iteration — shared with the outer `run` method so + /// `CompletionReport` can include accurate iteration counts. + iteration_tracker: Arc>, +} + +impl ContainerDelegate { + async fn post_event(&self, event_type: &str, data: serde_json::Value) { + self.client + .post_event(&JobEventPayload { + event_type: event_type.to_string(), + data, + }) + .await; + } + + /// Poll the orchestrator for a follow-up prompt. If one is available, + /// inject it as a user message into the reasoning context. + async fn poll_and_inject_prompt(&self, reason_ctx: &mut ReasoningContext) { + match self.client.poll_prompt().await { + Ok(Some(prompt)) => { + tracing::info!( + "Received follow-up prompt: {}", + truncate_for_preview(&prompt.content, 100) + ); + self.post_event( + "message", + serde_json::json!({ + "role": "user", + "content": truncate_for_preview(&prompt.content, 2000), + }), + ) + .await; + reason_ctx.messages.push(ChatMessage::user(&prompt.content)); + } + Ok(None) => {} + Err(e) => { + tracing::debug!("Failed to poll for prompt: {}", e); + } + } + } +} + +#[async_trait] +impl LoopDelegate for ContainerDelegate { + async fn check_signals(&self) -> LoopSignal { + // Container runtime has no stop signals — the orchestrator manages lifecycle. + LoopSignal::Continue + } + + async fn before_llm_call( + &self, + reason_ctx: &mut ReasoningContext, + iteration: usize, + ) -> Option { + let iteration = iteration as u32; + *self.iteration_tracker.lock().await = iteration; + + // Report progress every 5 iterations + if iteration % 5 == 1 { + let _ = self + .client + .report_status(&StatusUpdate { + state: "in_progress".to_string(), + message: Some(format!("Iteration {}", iteration)), + iteration, + }) + .await; + } + + // Poll for follow-up prompts from the user + self.poll_and_inject_prompt(reason_ctx).await; + + // Refresh tools (in case WASM tools were built) + reason_ctx.available_tools = self.tools.tool_definitions().await; + + None + } + + async fn call_llm( + &self, + reasoning: &Reasoning, + reason_ctx: &mut ReasoningContext, + _iteration: usize, + ) -> Result { + // Container uses respond_with_tools (which may return either text or tool calls) + reasoning + .respond_with_tools(reason_ctx) + .await + .map_err(Into::into) + } + + async fn handle_text_response( + &self, + text: &str, + reason_ctx: &mut ReasoningContext, + ) -> TextAction { + self.post_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": truncate_for_preview(text, 2000), + }), + ) + .await; + + // Check for completion + if crate::util::llm_signals_completion(text) { + let last = self.last_output.lock().await; + let output = if last.is_empty() { + text.to_string() + } else { + last.clone() + }; + return TextAction::Return(LoopOutcome::Response(output)); + } + + reason_ctx.messages.push(ChatMessage::assistant(text)); + TextAction::Continue + } + + async fn execute_tool_calls( + &self, + tool_calls: Vec, + content: Option, + reason_ctx: &mut ReasoningContext, + ) -> Result, crate::error::Error> { + if let Some(ref text) = content { + self.post_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": truncate_for_preview(text, 2000), + }), + ) + .await; + } + + // Add assistant message with tool_calls (OpenAI protocol) + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls( + content, + tool_calls.clone(), + )); + + // Execute tools sequentially (container context — no parallel execution) + for tc in tool_calls { + self.post_event( + "tool_use", + serde_json::json!({ + "tool_name": tc.name, + "input": truncate_for_preview(&tc.arguments.to_string(), 500), + }), + ) + .await; + + let job_ctx = JobContext { + extra_env: self.extra_env.clone(), + ..Default::default() + }; + + let result = + execute_tool_simple(&self.tools, &self.safety, &tc.name, &tc.arguments, &job_ctx) + .await; + + self.post_event( + "tool_result", + serde_json::json!({ + "tool_name": tc.name, + "output": match &result { + Ok(output) => truncate_for_preview(output, 2000), + Err(e) => format!("Error: {}", truncate_for_preview(e, 500)), + }, + "success": result.is_ok(), + }), + ) + .await; + + if let Ok(ref output) = result { + *self.last_output.lock().await = output.clone(); + } + + // Use shared result processing + let (_, message) = process_tool_result(&self.safety, &tc.name, &tc.id, &result); + reason_ctx.messages.push(message); + } + + Ok(None) + } + + async fn on_tool_intent_nudge(&self, text: &str, _reason_ctx: &mut ReasoningContext) { + self.post_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": truncate_for_preview(text, 2000), + "nudge": true, + }), + ) + .await; + } + + async fn after_iteration(&self, _iteration: usize) { + // Brief pause between iterations + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +#[cfg(test)] +mod tests { + use crate::agent::agentic_loop::truncate_for_preview; + + #[test] + fn test_truncate_within_limit() { + assert_eq!(truncate_for_preview("hello", 10), "hello"); + } + + #[test] + fn test_truncate_at_limit() { + assert_eq!(truncate_for_preview("hello", 5), "hello"); + } + + #[test] + fn test_truncate_beyond_limit() { + let result = truncate_for_preview("hello world", 5); + assert_eq!(result, "hello..."); + } + + #[test] + fn test_truncate_multibyte_safe() { + // "é" is 2 bytes in UTF-8; slicing at byte 1 would panic without safety + let result = truncate_for_preview("é is fancy", 1); + // Should truncate to 0 chars (can't fit "é" in 1 byte) + assert_eq!(result, "..."); + } +} diff --git a/src/agent/worker.rs b/src/worker/job.rs similarity index 72% rename from src/agent/worker.rs rename to src/worker/job.rs index 19bfc8e5..ad5c7157 100644 --- a/src/agent/worker.rs +++ b/src/worker/job.rs @@ -1,12 +1,21 @@ -//! Per-job worker execution. +//! Job worker execution via the shared `AgenticLoop`. +//! +//! Replaces `src/agent/worker.rs` with a `JobDelegate` that implements +//! `LoopDelegate`. The `Worker` struct and `WorkerDeps` remain as the +//! public API consumed by `scheduler.rs`. use std::sync::Arc; use std::time::Duration; +use async_trait::async_trait; use tokio::sync::mpsc; use tokio::task::JoinSet; use uuid::Uuid; +use crate::agent::agentic_loop::{ + AgenticLoopConfig, LoopDelegate, LoopOutcome, LoopSignal, TextAction, run_agentic_loop, + truncate_for_preview, +}; use crate::agent::scheduler::WorkerMessage; use crate::agent::task::TaskOutput; use crate::channels::web::types::SseEvent; @@ -19,6 +28,7 @@ use crate::llm::{ ToolSelection, }; use crate::safety::SafetyLayer; +use crate::tools::execute::process_tool_result; use crate::tools::rate_limiter::RateLimitResult; use crate::tools::{ApprovalContext, ToolRegistry, redact_params}; @@ -72,6 +82,7 @@ impl Worker { &self.deps.llm } + #[allow(dead_code)] fn safety(&self) -> &Arc { &self.deps.safety } @@ -212,7 +223,8 @@ impl Worker { let job_ctx = self.context_manager().get_context(self.job_id).await?; // Create reasoning engine - let reasoning = Reasoning::new(self.llm().clone()); + let reasoning = + Reasoning::new(self.llm().clone()).with_model_name(self.llm().active_model_name()); // Build initial reasoning context (tool definitions refreshed each iteration in execution_loop) let mut reason_ctx = ReasoningContext::new().with_job(&job_ctx.description); @@ -241,24 +253,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."# Ok(Ok(())) => { tracing::info!("Worker for job {} completed successfully", self.job_id); // Only mark completed if still in an active, non-stuck state. - // The execution_loop may have already called mark_completed or - // mark_stuck (e.g. "plan completed but work remains"). let current_state = self .context_manager() .get_context(self.job_id) .await .map(|ctx| ctx.state); match current_state { - Ok(state) if state.is_terminal() => { - // Already in a terminal state (e.g. execution_loop - // called mark_completed itself). - } - Ok(JobState::Completed) => { - // execution_loop already called mark_completed. - } + Ok(state) if state.is_terminal() => {} + Ok(JobState::Completed) => {} Ok(JobState::Stuck) => { - // execution_loop marked this as stuck (e.g. "plan - // completed but work remains"); leave for self-repair. tracing::info!( "Job {} returned Ok but is Stuck — leaving for self-repair", self.job_id @@ -303,11 +306,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .and_then(|ctx| ctx.metadata.get("max_iterations").and_then(|v| v.as_u64())) .unwrap_or(50) as usize; let max_iterations = max_iterations.min(MAX_WORKER_ITERATIONS); - let mut iteration = 0; - const MAX_CONSECUTIVE_RATE_LIMITS: usize = 10; - let mut consecutive_rate_limits = 0usize; - const MAX_TOOL_INTENT_NUDGES: u32 = 2; - let mut consecutive_tool_intent_nudges: u32 = 0; // Initial tool definitions for planning (will be refreshed in loop) reason_ctx.available_tools = self.tools().tool_definitions().await; @@ -358,16 +356,10 @@ Report when the job is complete or if you encounter issues you cannot resolve."# None }; - // If we have a plan, execute it. Two exit paths: - // 1. Plan ran to completion → job is Completed or needs continuation - // (check state and only fall through if not terminal) - // 2. Plan was interrupted by UserMessage → fall through to direct loop + // If we have a plan, execute it. if let Some(ref plan) = plan { self.execute_plan(rx, reasoning, reason_ctx, plan).await?; - // If the plan marked the job completed, terminal, or stuck, we're - // done. Only fall through to the direct selection loop if the - // plan was interrupted or explicitly left the job in-progress. if let Ok(ctx) = self.context_manager().get_context(self.job_id).await && (ctx.state.is_terminal() || ctx.state == JobState::Stuck @@ -377,282 +369,36 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } } - // Direct tool selection loop (also used as fallback after plan interruption) - loop { - // Check for stop signal and injected user messages - while let Ok(msg) = rx.try_recv() { - match msg { - WorkerMessage::Stop => { - tracing::debug!("Worker for job {} received stop signal", self.job_id); - return Ok(()); - } - WorkerMessage::Ping => { - tracing::trace!("Worker for job {} received ping", self.job_id); - } - WorkerMessage::Start => {} - WorkerMessage::UserMessage(content) => { - tracing::info!( - job_id = %self.job_id, - "Worker received follow-up user message" - ); - reason_ctx.messages.push(ChatMessage::user(&content)); - self.log_event( - "message", - serde_json::json!({ - "role": "user", - "content": content, - }), - ); - } - } - } + // Build the delegate and run the shared agentic loop + let delegate = JobDelegate { + worker: self, + rx: tokio::sync::Mutex::new(rx), + consecutive_rate_limits: std::sync::atomic::AtomicUsize::new(0), + }; - // Check for cancellation - if let Ok(ctx) = self.context_manager().get_context(self.job_id).await - && ctx.state == JobState::Cancelled - { - tracing::info!("Worker for job {} detected cancellation", self.job_id); - return Ok(()); - } + let config = AgenticLoopConfig { + max_iterations, + enable_tool_intent_nudge: true, + max_tool_intent_nudges: 2, + }; - iteration += 1; - if iteration > max_iterations { + let outcome = run_agentic_loop(&delegate, reasoning, reason_ctx, &config).await?; + + match outcome { + LoopOutcome::Response(_) => { + // Completion was already handled in handle_text_response via mark_completed + } + LoopOutcome::MaxIterations => { self.mark_failed("Maximum iterations exceeded: job hit the iteration cap") .await?; - return Ok(()); } - - // Refresh tool definitions so newly built tools become visible - reason_ctx.available_tools = self.tools().tool_definitions().await; - - // Select next tool(s) to use, with rate-limit retry. - let selections = match reasoning.select_tools(reason_ctx).await { - Ok(s) => s, - Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { - consecutive_rate_limits += 1; - let wait = retry_after.unwrap_or(Duration::from_secs(5)); - tracing::warn!( - job_id = %self.job_id, - wait_secs = wait.as_secs(), - attempt = consecutive_rate_limits, - "LLM rate limited during tool selection, backing off" - ); - if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS { - self.mark_failed("Persistent rate limiting: exceeded retry limit") - .await?; - return Ok(()); - } - self.log_event( - "status", - serde_json::json!({ - "message": format!("Rate limited, retrying in {}s ({}/{})...", - wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS), - }), - ); - tokio::time::sleep(wait).await; - continue; - } - Err(e) => return Err(e.into()), - }; - - if selections.is_empty() { - // No tools from select_tools, ask LLM directly (may still return tool calls) - let respond_output = match reasoning.respond_with_tools(reason_ctx).await { - Ok(o) => o, - Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { - consecutive_rate_limits += 1; - let wait = retry_after.unwrap_or(Duration::from_secs(5)); - tracing::warn!( - job_id = %self.job_id, - wait_secs = wait.as_secs(), - attempt = consecutive_rate_limits, - "LLM rate limited during respond_with_tools, backing off" - ); - if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS { - self.mark_failed("Persistent rate limiting: exceeded retry limit") - .await?; - return Ok(()); - } - self.log_event( - "status", - serde_json::json!({ - "message": format!("Rate limited, retrying in {}s ({}/{})...", - wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS), - }), - ); - tokio::time::sleep(wait).await; - continue; - } - Err(e) => return Err(e.into()), - }; - - // Track token usage from LLM call against the job budget. - // NOTE: select_tools() also makes LLM calls but doesn't expose - // TokenUsage; only respond_with_tools() usage is tracked here. - let total_tokens = respond_output.usage.total() as u64; - if total_tokens > 0 - && let Err(msg) = self - .context_manager() - .update_context(self.job_id, |ctx| ctx.add_tokens(total_tokens)) - .await? - { - self.mark_failed(&msg).await?; - return Ok(()); - } - - match respond_output.result { - RespondResult::Text(response) => { - // Check for explicit completion phrases. Use word-boundary - // aware checks to avoid false positives like "incomplete", - // "not done", or "unfinished". Only the LLM's own response - // (not tool output) can trigger this. - if crate::util::llm_signals_completion(&response) { - self.mark_completed().await?; - return Ok(()); - } - - // Add assistant response to context - reason_ctx.messages.push(ChatMessage::assistant(&response)); - - self.log_event( - "message", - serde_json::json!({ - "role": "assistant", - "content": response, - }), - ); - - // Nudge the LLM if it expressed tool intent without calling tools - let signals_intent = !reason_ctx.available_tools.is_empty() - && crate::llm::llm_signals_tool_intent(&response); - if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES - { - consecutive_tool_intent_nudges += 1; - tracing::info!( - job_id = %self.job_id, - "LLM expressed tool intent without calling a tool, nudging" - ); - reason_ctx - .messages - .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); - } else if !signals_intent { - consecutive_tool_intent_nudges = 0; - if iteration > 3 && iteration % 5 == 0 { - // Generic fallback nudge - reason_ctx.messages.push(ChatMessage::user( - "Are you stuck? Do you need help completing this job?", - )); - } - } - } - RespondResult::ToolCalls { - tool_calls, - content, - } => { - consecutive_tool_intent_nudges = 0; - // Model returned tool calls - execute them - tracing::debug!( - "Job {} respond_with_tools returned {} tool calls", - self.job_id, - tool_calls.len() - ); - - if let Some(ref text) = content { - self.log_event( - "message", - serde_json::json!({ - "role": "assistant", - "content": text, - }), - ); - } - - // Add assistant message with tool_calls (OpenAI protocol) - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); - - // Convert ToolCalls to ToolSelections and execute in parallel - let selections: Vec = tool_calls - .iter() - .map(|tc| ToolSelection { - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - reasoning: String::new(), - alternatives: vec![], - tool_call_id: tc.id.clone(), - }) - .collect(); - - let results = self.execute_tools_parallel(&selections).await; - for (selection, result) in selections.iter().zip(results) { - self.process_tool_result(reason_ctx, selection, result.result) - .await?; - } - } - } - } else { - consecutive_tool_intent_nudges = 0; - - // Record the assistant tool_calls message so that tool_result - // messages have a matching parent (prevents orphaned rewrites). - let tool_calls: Vec = selections - .iter() - .map(|s| ToolCall { - id: s.tool_call_id.clone(), - name: s.tool_name.clone(), - arguments: s.parameters.clone(), - }) - .collect(); - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls(None, tool_calls)); - - if selections.len() == 1 { - // Single tool: execute directly - let selection = &selections[0]; - tracing::debug!( - "Job {} selecting tool: {} - {}", - self.job_id, - selection.tool_name, - selection.reasoning - ); - - let result = self - .execute_tool(&selection.tool_name, &selection.parameters) - .await; - - self.process_tool_result(reason_ctx, selection, result) - .await?; - } else { - // Multiple tools: execute in parallel - tracing::debug!( - "Job {} executing {} tools in parallel", - self.job_id, - selections.len() - ); - - let results = self.execute_tools_parallel(&selections).await; - - // Process all results - for (selection, result) in selections.iter().zip(results) { - self.process_tool_result(reason_ctx, selection, result.result) - .await?; - } - } + LoopOutcome::Stopped => { + // Stop signal handled — nothing more to do } - - // Reset rate-limit counter after a successful iteration (all LLM - // calls succeeded). Placed here so alternating success/fail between - // select_tools and respond_with_tools cannot bypass the cap. - consecutive_rate_limits = 0; - - // Small delay between iterations - tokio::time::sleep(Duration::from_millis(100)).await; + LoopOutcome::NeedApproval(_) => {} } + + Ok(()) } /// Execute multiple tools in parallel using a JoinSet. @@ -832,8 +578,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .into()); } - // Redact sensitive parameter values (e.g. secret_save's "value") before - // they touch any observability or audit path. + // Redact sensitive parameter values before they touch any observability or audit path. let safe_params = redact_params(¶ms, tool.sensitive_params()); tracing::debug!( tool = %tool_name, @@ -853,12 +598,13 @@ Report when the job is complete or if you encounter issues you cannot resolve."# match &result { Ok(Ok(output)) => { - let result_str = serde_json::to_string(&output.result) - .unwrap_or_else(|_| "".to_string()); + let result_size = serde_json::to_string(&output.result) + .map(|s| s.len()) + .unwrap_or(0); tracing::debug!( tool = %tool_name, elapsed_ms = elapsed.as_millis() as u64, - result = %result_str, + result_size_bytes = result_size, "Tool call succeeded" ); } @@ -977,51 +723,47 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } /// Process a tool execution result and add it to the reasoning context. - async fn process_tool_result( + async fn process_tool_result_job( &self, reason_ctx: &mut ReasoningContext, selection: &ToolSelection, result: Result, - ) -> Result { + ) -> Result<(), Error> { self.log_event( "tool_use", serde_json::json!({ "tool_name": selection.tool_name, - "input": crate::agent::agent_loop::truncate_for_preview( + "input": truncate_for_preview( &selection.parameters.to_string(), 500), }), ); - match result { - Ok(output) => { - // Sanitize output + // Use shared result processing for sanitize → wrap → ChatMessage. + // The wrapped content (XML tags) goes into reason_ctx for the LLM. + // The raw sanitized content goes into events/SSE for human-readable UI. + let (_wrapped, message) = process_tool_result( + &self.deps.safety, + &selection.tool_name, + &selection.tool_call_id, + &result, + ); + reason_ctx.messages.push(message); + + match &result { + Ok(raw_output) => { let sanitized = self - .safety() - .sanitize_tool_output(&selection.tool_name, &output); - - // Add to context - let wrapped = self.safety().wrap_for_llm( - &selection.tool_name, - &sanitized.content, - sanitized.was_modified, + .deps + .safety + .sanitize_tool_output(&selection.tool_name, raw_output); + self.log_event( + "tool_result", + serde_json::json!({ + "tool_name": selection.tool_name, + "success": true, + "output": truncate_for_preview(&sanitized.content, 500), + }), ); - - reason_ctx.messages.push(ChatMessage::tool_result( - &selection.tool_call_id, - &selection.tool_name, - wrapped, - )); - - self.log_event("tool_result", serde_json::json!({ - "tool_name": selection.tool_name, - "success": true, - "output": crate::agent::agent_loop::truncate_for_preview(&sanitized.content, 500), - })); - - // Tool output never drives job completion. A malicious tool could - // emit "TASK_COMPLETE" to force premature completion. Only the LLM's - // own structured response (in execution_loop) can mark a job done. - Ok(false) + Ok(()) } Err(e) => { tracing::warn!( @@ -1049,17 +791,11 @@ Report when the job is complete or if you encounter issues you cannot resolve."# serde_json::json!({ "tool_name": selection.tool_name, "success": false, - "output": format!("Error: {}", e), + "output": truncate_for_preview(&format!("Error: {}", e), 500), }), ); - reason_ctx.messages.push(ChatMessage::tool_result( - &selection.tool_call_id, - &selection.tool_name, - format!("Error: {}", e), - )); - - Ok(false) + Ok(()) } } } @@ -1106,8 +842,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# "message": "Plan interrupted by user message, re-evaluating...", }), ); - // Return Ok to break out of plan; caller falls through to - // the direct selection loop for LLM re-evaluation. return Ok(()); } } @@ -1122,9 +856,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# action.reasoning ); - // Create a synthetic ToolSelection for process_tool_result. - // Plan actions don't originate from an LLM tool_call response so - // there is no real tool_call_id; generate a unique one. let selection = ToolSelection { tool_name: action.tool_name.clone(), parameters: action.parameters.clone(), @@ -1133,8 +864,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tool_call_id: format!("plan_{}_{}", self.job_id, i), }; - // Record the assistant tool_calls message so that the tool_result - // has a matching parent (prevents orphaned rewrites). reason_ctx .messages .push(ChatMessage::assistant_with_tool_calls( @@ -1146,21 +875,13 @@ Report when the job is complete or if you encounter issues you cannot resolve."# }], )); - // Execute the planned tool let result = self .execute_tool(&action.tool_name, &action.parameters) .await; - // Process the result - let completed = self - .process_tool_result(reason_ctx, &selection, result) + self.process_tool_result_job(reason_ctx, &selection, result) .await?; - if completed { - return Ok(()); - } - - // Small delay between actions tokio::time::sleep(Duration::from_millis(100)).await; } @@ -1175,8 +896,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# if crate::util::llm_signals_completion(&response) { self.mark_completed().await?; } else { - // Job not complete — return Ok without marking terminal so the - // caller falls through to the direct selection loop for continuation. tracing::info!( "Job {} plan completed but work remains, falling back to direct selection", self.job_id @@ -1274,6 +993,343 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } } +/// Job delegate: implements `LoopDelegate` for the background job context. +/// +/// Handles: signal channel (stop/ping/user messages), cancellation checks, +/// rate-limit retry, parallel tool execution, DB persistence, SSE broadcasting. +struct JobDelegate<'a> { + worker: &'a Worker, + rx: tokio::sync::Mutex<&'a mut mpsc::Receiver>, + /// Tracks consecutive rate-limit errors to fail fast instead of burning iterations. + consecutive_rate_limits: std::sync::atomic::AtomicUsize, +} + +impl<'a> JobDelegate<'a> { + const MAX_CONSECUTIVE_RATE_LIMITS: usize = 10; + + /// Handle a rate-limit error: back off, increment counter, and fail fast + /// if the provider remains rate-limited for too many consecutive attempts. + async fn handle_rate_limit( + &self, + retry_after: Option, + context: &str, + ) -> Result { + use std::sync::atomic::Ordering::Relaxed; + + let count = self.consecutive_rate_limits.fetch_add(1, Relaxed) + 1; + let wait = retry_after.unwrap_or(Duration::from_secs(5)); + tracing::warn!( + job_id = %self.worker.job_id, + wait_secs = wait.as_secs(), + attempt = count, + "LLM rate limited during {}, backing off", + context, + ); + + if count >= Self::MAX_CONSECUTIVE_RATE_LIMITS { + self.worker + .mark_failed("Persistent rate limiting: exceeded retry limit") + .await?; + return Err(crate::error::LlmError::RateLimited { + provider: "rate-limit-exhausted".to_string(), + retry_after: None, + } + .into()); + } + + self.worker.log_event( + "status", + serde_json::json!({ + "message": format!( + "Rate limited, retrying in {}s... ({}/{})", + wait.as_secs(), count, Self::MAX_CONSECUTIVE_RATE_LIMITS + ), + }), + ); + tokio::time::sleep(wait).await; + + Ok(crate::llm::RespondOutput { + result: RespondResult::Text(String::new()), + usage: crate::llm::TokenUsage::default(), + }) + } +} + +#[async_trait] +impl<'a> LoopDelegate for JobDelegate<'a> { + async fn check_signals(&self) -> LoopSignal { + // Drain the entire message channel, prioritizing Stop over user messages. + // Scope the lock so it's dropped before any .await below. + let mut stop_requested = false; + let mut first_user_message: Option = None; + { + let mut rx = self.rx.lock().await; + while let Ok(msg) = rx.try_recv() { + match msg { + WorkerMessage::Stop => { + tracing::debug!( + "Worker for job {} received stop signal", + self.worker.job_id + ); + stop_requested = true; + } + WorkerMessage::Ping => { + tracing::trace!("Worker for job {} received ping", self.worker.job_id); + } + WorkerMessage::Start => {} + WorkerMessage::UserMessage(content) => { + tracing::info!( + job_id = %self.worker.job_id, + "Worker received follow-up user message" + ); + self.worker.log_event( + "message", + serde_json::json!({ + "role": "user", + "content": content, + }), + ); + // Keep only the first user message; subsequent ones will be + // picked up on the next iteration's drain. + if first_user_message.is_none() { + first_user_message = Some(content); + } + } + } + } + } // MutexGuard dropped here, before the cancellation .await + + // Stop takes priority over user messages + if stop_requested { + return LoopSignal::Stop; + } + + if let Some(content) = first_user_message { + return LoopSignal::InjectMessage(content); + } + + // Check for terminal or non-progressing state. The loop should stop when the + // job has been cancelled, failed, stuck, or already completed — not just the + // three states that `is_terminal()` covers (Accepted/Failed/Cancelled). + if let Ok(ctx) = self + .worker + .context_manager() + .get_context(self.worker.job_id) + .await + && matches!( + ctx.state, + JobState::Cancelled + | JobState::Failed + | JobState::Stuck + | JobState::Completed + | JobState::Submitted + | JobState::Accepted + ) + { + tracing::info!( + "Worker for job {} detected terminal state {:?}", + self.worker.job_id, + ctx.state, + ); + return LoopSignal::Stop; + } + + LoopSignal::Continue + } + + async fn before_llm_call( + &self, + reason_ctx: &mut ReasoningContext, + _iteration: usize, + ) -> Option { + // Refresh tool definitions so newly built tools become visible + reason_ctx.available_tools = self.worker.tools().tool_definitions().await; + None + } + + async fn call_llm( + &self, + reasoning: &Reasoning, + reason_ctx: &mut ReasoningContext, + _iteration: usize, + ) -> Result { + // Try select_tools first, fall back to respond_with_tools + match reasoning.select_tools(reason_ctx).await { + Ok(s) if !s.is_empty() => { + // Reset counter after a successful LLM call + self.consecutive_rate_limits + .store(0, std::sync::atomic::Ordering::Relaxed); + let tool_calls: Vec = selections_to_tool_calls(&s); + return Ok(crate::llm::RespondOutput { + result: RespondResult::ToolCalls { + tool_calls, + content: None, + }, + usage: crate::llm::TokenUsage::default(), + }); + } + Ok(_) => {} // empty selections, fall through + Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { + return self.handle_rate_limit(retry_after, "tool selection").await; + } + Err(e) => return Err(e.into()), + }; + + // Fall back to respond_with_tools + match reasoning.respond_with_tools(reason_ctx).await { + Ok(output) => { + // Reset counter after a successful LLM call + self.consecutive_rate_limits + .store(0, std::sync::atomic::Ordering::Relaxed); + + // Track token usage against the job budget. + // NOTE: select_tools() also makes LLM calls but doesn't expose + // TokenUsage; only respond_with_tools() usage is tracked here. + let total_tokens = output.usage.total() as u64; + if total_tokens > 0 + && let Err(msg) = self + .worker + .context_manager() + .update_context(self.worker.job_id, |ctx| ctx.add_tokens(total_tokens)) + .await? + { + self.worker.mark_failed(&msg).await?; + } + + Ok(output) + } + Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { + self.handle_rate_limit(retry_after, "respond_with_tools") + .await + } + Err(e) => Err(e.into()), + } + } + + async fn handle_text_response( + &self, + text: &str, + reason_ctx: &mut ReasoningContext, + ) -> TextAction { + // Empty text from rate-limit backoff retry — skip processing and let the + // loop proceed to the next iteration which will re-call the LLM. + if text.is_empty() { + return TextAction::Continue; + } + + // Check for explicit completion + if crate::util::llm_signals_completion(text) { + if let Err(e) = self.worker.mark_completed().await { + tracing::warn!( + "Failed to mark job {} as completed: {}", + self.worker.job_id, + e + ); + } + return TextAction::Return(LoopOutcome::Response(text.to_string())); + } + + // Add assistant response to context + reason_ctx.messages.push(ChatMessage::assistant(text)); + + self.worker.log_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": text, + }), + ); + + TextAction::Continue + } + + async fn execute_tool_calls( + &self, + tool_calls: Vec, + content: Option, + reason_ctx: &mut ReasoningContext, + ) -> Result, crate::error::Error> { + if let Some(ref text) = content { + self.worker.log_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": text, + }), + ); + } + + // Add assistant message with tool_calls (OpenAI protocol) + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls( + content, + tool_calls.clone(), + )); + + // Convert to ToolSelections + let selections: Vec = tool_calls + .iter() + .map(|tc| ToolSelection { + tool_name: tc.name.clone(), + parameters: tc.arguments.clone(), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: tc.id.clone(), + }) + .collect(); + + // Execute tools (parallel for multiple, direct for single) + if selections.len() == 1 { + let selection = &selections[0]; + let result = self + .worker + .execute_tool(&selection.tool_name, &selection.parameters) + .await; + self.worker + .process_tool_result_job(reason_ctx, selection, result) + .await?; + } else { + let results = self.worker.execute_tools_parallel(&selections).await; + for (selection, result) in selections.iter().zip(results) { + self.worker + .process_tool_result_job(reason_ctx, selection, result.result) + .await?; + } + } + + Ok(None) + } + + async fn on_tool_intent_nudge(&self, text: &str, _reason_ctx: &mut ReasoningContext) { + self.worker.log_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": truncate_for_preview(text, 2000), + "nudge": true, + }), + ); + } + + async fn after_iteration(&self, _iteration: usize) { + // Small delay between iterations + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// Convert `ToolSelection`s to `ToolCall`s. +fn selections_to_tool_calls(selections: &[ToolSelection]) -> Vec { + selections + .iter() + .map(|s| ToolCall { + id: s.tool_call_id.clone(), + name: s.tool_name.clone(), + arguments: s.parameters.clone(), + }) + .collect() +} + /// Convert a TaskOutput to a string result for tool execution. impl From for Result { fn from(output: TaskOutput) -> Self { @@ -1290,7 +1346,6 @@ impl From for Result { #[cfg(test)] mod tests { use crate::llm::ToolSelection; - use crate::util::llm_signals_completion; use super::*; use crate::config::SafetyConfig; @@ -1300,7 +1355,7 @@ mod tests { ToolCompletionResponse, }; use crate::safety::SafetyLayer; - use crate::tools::{Tool, ToolError, ToolOutput}; + use crate::tools::{Tool, ToolError as ToolExecError, ToolOutput}; /// A test tool that sleeps for a configurable duration before returning. struct SlowTool { @@ -1323,7 +1378,7 @@ mod tests { &self, _params: serde_json::Value, _ctx: &JobContext, - ) -> Result { + ) -> Result { let start = std::time::Instant::now(); tokio::time::sleep(self.delay).await; Ok(ToolOutput::text( @@ -1408,70 +1463,11 @@ mod tests { ); } - #[test] - fn test_completion_positive_signals() { - assert!(llm_signals_completion("The job is complete.")); - assert!(llm_signals_completion( - "I have completed the task successfully." - )); - assert!(llm_signals_completion("The task is done.")); - assert!(llm_signals_completion("The task is finished.")); - assert!(llm_signals_completion( - "All steps are complete and verified." - )); - assert!(llm_signals_completion( - "I've done all the work. The work is done." - )); - assert!(llm_signals_completion( - "Successfully completed the migration." - )); - } - - #[test] - fn test_completion_negative_signals_block_false_positives() { - // These contain completion keywords but also negation, should NOT trigger. - assert!(!llm_signals_completion("The task is not complete yet.")); - assert!(!llm_signals_completion("This is not done.")); - assert!(!llm_signals_completion("The work is incomplete.")); - assert!(!llm_signals_completion( - "The migration is not yet finished." - )); - assert!(!llm_signals_completion("The job isn't done yet.")); - assert!(!llm_signals_completion("This remains unfinished.")); - } - - #[test] - fn test_completion_does_not_match_bare_substrings() { - // Bare words embedded in other text should NOT trigger completion. - assert!(!llm_signals_completion( - "I need to complete more work first." - )); - assert!(!llm_signals_completion( - "Let me finish the remaining steps." - )); - assert!(!llm_signals_completion( - "I'm done analyzing, now let me fix it." - )); - assert!(!llm_signals_completion( - "I completed step 1 but step 2 remains." - )); - } - - #[test] - fn test_completion_tool_output_injection() { - // A malicious tool output echoed by the LLM should not trigger - // completion unless it forms a genuine completion phrase. - assert!(!llm_signals_completion("TASK_COMPLETE")); - assert!(!llm_signals_completion("JOB_DONE")); - assert!(!llm_signals_completion( - "The tool returned: TASK_COMPLETE signal" - )); - } + // Completion detection tests live in src/util.rs (the canonical location). + // See: test_completion_signals, test_completion_negative, etc. #[tokio::test] async fn test_parallel_speedup() { - // 3 tools each sleeping 200ms should finish in roughly 200ms (parallel), - // not ~600ms (sequential). let tools: Vec> = (0..3) .map(|i| { Arc::new(SlowTool { @@ -1501,9 +1497,6 @@ mod tests { for r in &results { assert!(r.result.is_ok(), "Tool should succeed"); } - // Parallel should complete well under the sequential 600ms threshold. - // Use a generous bound (800ms) to avoid flaky failures on slow CI runners, - // while still proving parallelism (sequential would be >= 600ms on any machine). assert!( elapsed < Duration::from_millis(800), "Parallel execution took {:?}, expected < 800ms (sequential would be ~600ms)", @@ -1513,8 +1506,6 @@ mod tests { #[tokio::test] async fn test_result_ordering_preserved() { - // Tools with different delays finish in different order. - // Results must be returned in the original request order. let tools: Vec> = vec![ Arc::new(SlowTool { tool_name: "tool_a".into(), @@ -1558,7 +1549,6 @@ mod tests { let results = worker.execute_tools_parallel(&selections).await; - // Results must be in same order as selections, not completion order. assert!(results[0].result.as_ref().unwrap().contains("done_tool_a")); assert!(results[1].result.as_ref().unwrap().contains("done_tool_b")); assert!(results[2].result.as_ref().unwrap().contains("done_tool_c")); @@ -1566,7 +1556,6 @@ mod tests { #[tokio::test] async fn test_missing_tool_produces_error_not_panic() { - // If a tool doesn't exist, the result slot should contain an error. let worker = make_worker(vec![]).await; let selections = vec![ToolSelection { @@ -1585,13 +1574,10 @@ mod tests { ); } - /// Verify that calling mark_completed on an already-Completed job returns - /// an error (Completed → Completed is an invalid state transition). #[tokio::test] async fn test_mark_completed_twice_returns_error() { let worker = make_worker(vec![]).await; - // Transition to InProgress first (required by state machine) worker .context_manager() .update_context(worker.job_id, |ctx| { @@ -1601,10 +1587,8 @@ mod tests { .unwrap() .unwrap(); - // First mark_completed should succeed worker.mark_completed().await.unwrap(); - // Verify state is Completed let ctx = worker .context_manager() .get_context(worker.job_id) @@ -1612,7 +1596,6 @@ mod tests { .unwrap(); assert_eq!(ctx.state, JobState::Completed); - // Second mark_completed should fail (Completed → Completed is invalid) let result = worker.mark_completed().await; assert!( result.is_err(), @@ -1725,7 +1708,6 @@ mod tests { #[tokio::test] async fn test_approval_context_unblocks_unless_auto_approved() { - // Without approval context, UnlessAutoApproved is blocked let worker_blocked = make_worker_with_approval(vec![Arc::new(ApprovalTool)], None).await; let result = worker_blocked .execute_tool("needs_approval", &serde_json::json!({})) @@ -1735,7 +1717,6 @@ mod tests { "Should be blocked without approval context" ); - // With autonomous approval context, UnlessAutoApproved is allowed let worker_allowed = make_worker_with_approval( vec![Arc::new(ApprovalTool)], Some(crate::tools::ApprovalContext::autonomous()), @@ -1749,7 +1730,6 @@ mod tests { #[tokio::test] async fn test_approval_context_blocks_always_unless_permitted() { - // Autonomous context without tool_permissions blocks Always tools let worker_blocked = make_worker_with_approval( vec![Arc::new(AlwaysApprovalTool)], Some(crate::tools::ApprovalContext::autonomous()), @@ -1763,7 +1743,6 @@ mod tests { "Always tool should be blocked without permission" ); - // Autonomous context with tool_permissions allows Always tools let worker_allowed = make_worker_with_approval( vec![Arc::new(AlwaysApprovalTool)], Some(crate::tools::ApprovalContext::autonomous_with_tools([ diff --git a/src/worker/mod.rs b/src/worker/mod.rs index dce75b3d..c6028b96 100644 --- a/src/worker/mod.rs +++ b/src/worker/mod.rs @@ -26,13 +26,15 @@ pub mod api; pub mod claude_bridge; +pub mod container; +pub mod job; pub mod proxy_llm; -pub mod runtime; pub use api::WorkerHttpClient; pub use claude_bridge::ClaudeBridgeRuntime; +pub use container::WorkerRuntime; +pub use job::{Worker, WorkerDeps}; pub use proxy_llm::ProxyLlmProvider; -pub use runtime::WorkerRuntime; /// Run the Worker subcommand (inside Docker containers). pub async fn run_worker( @@ -46,7 +48,7 @@ pub async fn run_worker( orchestrator_url ); - let config = runtime::WorkerConfig { + let config = container::WorkerConfig { job_id, orchestrator_url: orchestrator_url.to_string(), max_iterations, diff --git a/src/worker/runtime.rs b/src/worker/runtime.rs deleted file mode 100644 index 5dd00e5a..00000000 --- a/src/worker/runtime.rs +++ /dev/null @@ -1,569 +0,0 @@ -//! Worker runtime: the main execution loop inside a container. -//! -//! Reuses the existing `Reasoning` and `SafetyLayer` infrastructure but -//! connects to the orchestrator for LLM calls instead of calling APIs directly. -//! Streams real-time events (message, tool_use, tool_result, result) through -//! the orchestrator's job event pipeline for UI visibility. - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use uuid::Uuid; - -use crate::config::SafetyConfig; -use crate::context::JobContext; -use crate::error::WorkerError; -use crate::llm::{ - ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection, -}; -use crate::safety::SafetyLayer; -use crate::tools::ToolRegistry; -use crate::worker::api::{CompletionReport, JobEventPayload, StatusUpdate, WorkerHttpClient}; -use crate::worker::proxy_llm::ProxyLlmProvider; - -/// Configuration for the worker runtime. -pub struct WorkerConfig { - pub job_id: Uuid, - pub orchestrator_url: String, - pub max_iterations: u32, - pub timeout: Duration, -} - -impl Default for WorkerConfig { - fn default() -> Self { - Self { - job_id: Uuid::nil(), - orchestrator_url: String::new(), - max_iterations: 50, - timeout: Duration::from_secs(600), - } - } -} - -/// The worker runtime runs inside a Docker container. -/// -/// It connects to the orchestrator over HTTP, fetches its job description, -/// then runs a tool execution loop until the job is complete. Events are -/// streamed to the orchestrator so the UI can show real-time progress. -pub struct WorkerRuntime { - config: WorkerConfig, - client: Arc, - llm: Arc, - safety: Arc, - tools: Arc, - /// Credentials fetched from the orchestrator, injected into child processes - /// via `Command::envs()` rather than mutating the global process environment. - /// - /// Wrapped in `Arc` to avoid deep-cloning the map on every tool invocation. - extra_env: Arc>, -} - -impl WorkerRuntime { - /// Create a new worker runtime. - /// - /// Reads `IRONCLAW_WORKER_TOKEN` from the environment for auth. - pub fn new(config: WorkerConfig) -> Result { - let client = Arc::new(WorkerHttpClient::from_env( - config.orchestrator_url.clone(), - config.job_id, - )?); - - let llm: Arc = Arc::new(ProxyLlmProvider::new( - Arc::clone(&client), - "proxied".to_string(), - )); - - let safety = Arc::new(SafetyLayer::new(&SafetyConfig { - max_output_length: 100_000, - injection_check_enabled: true, - })); - - let tools = Arc::new(ToolRegistry::new()); - // Register only container-safe tools - tools.register_container_tools(); - - Ok(Self { - config, - client, - llm, - safety, - tools, - extra_env: Arc::new(HashMap::new()), - }) - } - - /// Run the worker until the job is complete or an error occurs. - pub async fn run(mut self) -> Result<(), WorkerError> { - tracing::info!("Worker starting for job {}", self.config.job_id); - - // Fetch job description from orchestrator - let job = self.client.get_job().await?; - - tracing::info!( - "Received job: {} - {}", - job.title, - truncate(&job.description, 100) - ); - - // Fetch credentials and store them for injection into child processes - // via Command::envs() (avoids unsafe std::env::set_var in multi-threaded runtime). - let credentials = self.client.fetch_credentials().await?; - { - let mut env_map = HashMap::new(); - for cred in &credentials { - env_map.insert(cred.env_var.clone(), cred.value.clone()); - } - self.extra_env = Arc::new(env_map); - } - if !credentials.is_empty() { - tracing::info!( - "Fetched {} credential(s) for child process injection", - credentials.len() - ); - } - - // Report that we're starting - self.client - .report_status(&StatusUpdate { - state: "in_progress".to_string(), - message: Some("Worker started, beginning execution".to_string()), - iteration: 0, - }) - .await?; - - // Create reasoning engine - let reasoning = Reasoning::new(self.llm.clone()); - - // Build initial context - let mut reason_ctx = ReasoningContext::new().with_job(&job.description); - - reason_ctx.messages.push(ChatMessage::system(format!( - r#"You are an autonomous agent running inside a Docker container. - -Job: {} -Description: {} - -You have tools for shell commands, file operations, and code editing. -Work independently to complete this job. Report when done."#, - job.title, job.description - ))); - - // Run with timeout - let result = tokio::time::timeout(self.config.timeout, async { - self.execution_loop(&reasoning, &mut reason_ctx).await - }) - .await; - - match result { - Ok(Ok(output)) => { - tracing::info!("Worker completed job {} successfully", self.config.job_id); - self.post_event( - "result", - serde_json::json!({ - "success": true, - "message": truncate(&output, 2000), - }), - ) - .await; - self.client - .report_complete(&CompletionReport { - success: true, - message: Some(output), - iterations: 0, - }) - .await?; - } - Ok(Err(e)) => { - tracing::error!("Worker failed for job {}: {}", self.config.job_id, e); - self.post_event( - "result", - serde_json::json!({ - "success": false, - "message": format!("Execution failed: {}", e), - }), - ) - .await; - self.client - .report_complete(&CompletionReport { - success: false, - message: Some(format!("Execution failed: {}", e)), - iterations: 0, - }) - .await?; - } - Err(_) => { - tracing::warn!("Worker timed out for job {}", self.config.job_id); - self.post_event( - "result", - serde_json::json!({ - "success": false, - "message": "Execution timed out", - }), - ) - .await; - self.client - .report_complete(&CompletionReport { - success: false, - message: Some("Execution timed out".to_string()), - iterations: 0, - }) - .await?; - } - } - - Ok(()) - } - - async fn execution_loop( - &self, - reasoning: &Reasoning, - reason_ctx: &mut ReasoningContext, - ) -> Result { - let max_iterations = self.config.max_iterations; - let mut last_output = String::new(); - const MAX_TOOL_INTENT_NUDGES: u32 = 2; - let mut consecutive_tool_intent_nudges: u32 = 0; - - // Load tool definitions - reason_ctx.available_tools = self.tools.tool_definitions().await; - - for iteration in 1..=max_iterations { - // Report progress - if iteration % 5 == 1 { - let _ = self - .client - .report_status(&StatusUpdate { - state: "in_progress".to_string(), - message: Some(format!("Iteration {}", iteration)), - iteration, - }) - .await; - } - - // Poll for follow-up prompts from the user - self.poll_and_inject_prompt(reason_ctx).await; - - // Refresh tools (in case WASM tools were built) - reason_ctx.available_tools = self.tools.tool_definitions().await; - - // Ask the LLM what to do next - let selections = reasoning.select_tools(reason_ctx).await.map_err(|e| { - WorkerError::ExecutionFailed { - reason: format!("tool selection failed: {}", e), - } - })?; - - if selections.is_empty() { - // No tools selected, try direct response - let respond_result = - reasoning - .respond_with_tools(reason_ctx) - .await - .map_err(|e| WorkerError::ExecutionFailed { - reason: format!("respond_with_tools failed: {}", e), - })?; - - match respond_result.result { - RespondResult::Text(response) => { - self.post_event( - "message", - serde_json::json!({ - "role": "assistant", - "content": truncate(&response, 2000), - }), - ) - .await; - - if crate::util::llm_signals_completion(&response) { - if last_output.is_empty() { - last_output = response.clone(); - } - return Ok(last_output); - } - reason_ctx.messages.push(ChatMessage::assistant(&response)); - - // Nudge the LLM if it expressed tool intent without calling tools - let signals_intent = !reason_ctx.available_tools.is_empty() - && crate::llm::llm_signals_tool_intent(&response); - if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES - { - consecutive_tool_intent_nudges += 1; - tracing::info!( - "LLM expressed tool intent without calling a tool, nudging" - ); - reason_ctx - .messages - .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); - } else if !signals_intent { - consecutive_tool_intent_nudges = 0; - } - } - RespondResult::ToolCalls { - tool_calls, - content, - } => { - consecutive_tool_intent_nudges = 0; - if let Some(ref text) = content { - self.post_event( - "message", - serde_json::json!({ - "role": "assistant", - "content": truncate(text, 2000), - }), - ) - .await; - } - - // Add assistant message with tool_calls (OpenAI protocol) - reason_ctx - .messages - .push(ChatMessage::assistant_with_tool_calls( - content, - tool_calls.clone(), - )); - - for tc in tool_calls { - self.post_event( - "tool_use", - serde_json::json!({ - "tool_name": tc.name, - "input": truncate(&tc.arguments.to_string(), 500), - }), - ) - .await; - - let result = self.execute_tool(&tc.name, &tc.arguments).await; - - self.post_event( - "tool_result", - serde_json::json!({ - "tool_name": tc.name, - "output": match &result { - Ok(output) => truncate(output, 2000), - Err(e) => format!("Error: {}", truncate(e, 500)), - }, - "success": result.is_ok(), - }), - ) - .await; - - if let Ok(ref output) = result { - last_output = output.clone(); - } - let selection = ToolSelection { - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - reasoning: String::new(), - alternatives: vec![], - tool_call_id: tc.id.clone(), - }; - self.process_result(reason_ctx, &selection, result); - } - } - } - } else { - consecutive_tool_intent_nudges = 0; - // Execute selected tools - for selection in &selections { - self.post_event( - "tool_use", - serde_json::json!({ - "tool_name": selection.tool_name, - "input": truncate(&selection.parameters.to_string(), 500), - }), - ) - .await; - - let result = self - .execute_tool(&selection.tool_name, &selection.parameters) - .await; - - self.post_event( - "tool_result", - serde_json::json!({ - "tool_name": selection.tool_name, - "output": match &result { - Ok(output) => truncate(output, 2000), - Err(e) => format!("Error: {}", truncate(e, 500)), - }, - "success": result.is_ok(), - }), - ) - .await; - - if let Ok(ref output) = result { - last_output = output.clone(); - } - - let completed = self.process_result(reason_ctx, selection, result); - if completed { - return Ok(last_output); - } - } - } - - // Brief pause between iterations - tokio::time::sleep(Duration::from_millis(100)).await; - } - - Err(WorkerError::ExecutionFailed { - reason: format!("max iterations ({}) exceeded", max_iterations), - }) - } - - async fn execute_tool( - &self, - tool_name: &str, - params: &serde_json::Value, - ) -> Result { - let tool = match self.tools.get(tool_name).await { - Some(t) => t, - None => return Err(format!("tool '{}' not found", tool_name)), - }; - - let ctx = JobContext { - extra_env: self.extra_env.clone(), - ..Default::default() - }; - - // Validate params - let validation = self.safety.validator().validate_tool_params(params); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::>() - .join("; "); - return Err(format!("invalid parameters: {}", details)); - } - - // Execute with per-tool timeout - let tool_timeout = tool.execution_timeout(); - let result = tokio::time::timeout(tool_timeout, tool.execute(params.clone(), &ctx)).await; - - match result { - Ok(Ok(output)) => serde_json::to_string_pretty(&output.result) - .map_err(|e| format!("serialization error: {}", e)), - Ok(Err(e)) => Err(e.to_string()), - Err(_) => Err("tool execution timed out".to_string()), - } - } - - /// Process a tool result into the reasoning context. Returns true if the job is complete. - fn process_result( - &self, - reason_ctx: &mut ReasoningContext, - selection: &ToolSelection, - result: Result, - ) -> bool { - match result { - Ok(output) => { - let sanitized = self - .safety - .sanitize_tool_output(&selection.tool_name, &output); - let wrapped = self.safety.wrap_for_llm( - &selection.tool_name, - &sanitized.content, - sanitized.was_modified, - ); - - reason_ctx.messages.push(ChatMessage::tool_result( - &selection.tool_call_id, - &selection.tool_name, - wrapped, - )); - - // Tool output should never signal job completion. Only the LLM's - // natural language response should decide when a job is done. A - // tool could return text containing "TASK_COMPLETE" in its output - // (e.g. from file contents) and trigger a false positive. - false - } - Err(e) => { - tracing::warn!("Tool {} failed: {}", selection.tool_name, e); - reason_ctx.messages.push(ChatMessage::tool_result( - &selection.tool_call_id, - &selection.tool_name, - format!("Error: {}", e), - )); - false - } - } - } - - /// Post a job event to the orchestrator (fire-and-forget). - async fn post_event(&self, event_type: &str, data: serde_json::Value) { - self.client - .post_event(&JobEventPayload { - event_type: event_type.to_string(), - data, - }) - .await; - } - - /// Poll the orchestrator for a follow-up prompt. If one is available, - /// inject it as a user message into the reasoning context. - async fn poll_and_inject_prompt(&self, reason_ctx: &mut ReasoningContext) { - match self.client.poll_prompt().await { - Ok(Some(prompt)) => { - tracing::info!( - "Received follow-up prompt: {}", - truncate(&prompt.content, 100) - ); - self.post_event( - "message", - serde_json::json!({ - "role": "user", - "content": truncate(&prompt.content, 2000), - }), - ) - .await; - reason_ctx.messages.push(ChatMessage::user(&prompt.content)); - } - Ok(None) => {} - Err(e) => { - tracing::debug!("Failed to poll for prompt: {}", e); - } - } - } -} - -fn truncate(s: &str, max: usize) -> String { - if s.len() <= max { - s.to_string() - } else { - let end = crate::util::floor_char_boundary(s, max); - format!("{}...", &s[..end]) - } -} - -#[cfg(test)] -mod tests { - use crate::worker::runtime::truncate; - - #[test] - fn test_truncate_within_limit() { - assert_eq!(truncate("hello", 10), "hello"); - } - - #[test] - fn test_truncate_at_limit() { - assert_eq!(truncate("hello", 5), "hello"); - } - - #[test] - fn test_truncate_beyond_limit() { - let result = truncate("hello world", 5); - assert_eq!(result, "hello..."); - } - - #[test] - fn test_truncate_multibyte_safe() { - // "é" is 2 bytes in UTF-8; slicing at byte 1 would panic without safety - let result = truncate("é is fancy", 1); - // Should truncate to 0 chars (can't fit "é" in 1 byte) - assert_eq!(result, "..."); - } -} diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index 16c7bc0e..fa48072b 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -887,13 +887,13 @@ impl Workspace { Ok(_) => continue, Err(WorkspaceError::DocumentNotFound { .. }) => {} Err(e) => { - tracing::warn!("Failed to check {}: {}", path, e); + tracing::debug!("Failed to check {}: {}", path, e); continue; } } if let Err(e) = self.write(path, content).await { - tracing::warn!("Failed to seed {}: {}", path, e); + tracing::debug!("Failed to seed {}: {}", path, e); } else { count += 1; } @@ -977,7 +977,7 @@ impl Workspace { Ok(_) => continue, Err(WorkspaceError::DocumentNotFound { .. }) => {} Err(e) => { - tracing::warn!("Failed to check {}: {}", file_name, e); + tracing::trace!("Failed to check {}: {}", file_name, e); continue; } } diff --git a/tests/config_round_trip.rs b/tests/config_round_trip.rs index 9ae1e3a1..8351ff74 100644 --- a/tests/config_round_trip.rs +++ b/tests/config_round_trip.rs @@ -12,6 +12,11 @@ use tempfile::tempdir; use ironclaw::bootstrap::{save_bootstrap_env_to, upsert_bootstrap_var_to}; +/// Fake OpenAI API key for test use only. Mirrors the internal +/// `TEST_OPENAI_API_KEY_LONG` constant from the main crate, which is not +/// directly available to integration tests due to `#[cfg(test)]`. +const TEST_OPENAI_API_KEY_LONG: &str = "sk-test-key-1234567890"; + /// Parse a .env file into a HashMap using dotenvy. fn read_env_map(path: &std::path::Path) -> HashMap { dotenvy::from_path_iter(path) @@ -77,7 +82,7 @@ fn bootstrap_env_round_trips_embedding_disabled() { &[ ("DATABASE_BACKEND", "libsql"), ("EMBEDDING_ENABLED", "false"), - ("OPENAI_API_KEY", "sk-test-key-1234567890"), + ("OPENAI_API_KEY", TEST_OPENAI_API_KEY_LONG), ("ONBOARD_COMPLETED", "true"), ], ) @@ -92,7 +97,7 @@ fn bootstrap_env_round_trips_embedding_disabled() { ); assert_eq!( map.get("OPENAI_API_KEY").map(String::as_str), - Some("sk-test-key-1234567890"), + Some(TEST_OPENAI_API_KEY_LONG), "OPENAI_API_KEY must be preserved alongside EMBEDDING_ENABLED" ); } diff --git a/tests/e2e_builtin_tool_coverage.rs b/tests/e2e_builtin_tool_coverage.rs index c5ce339b..4387ebc5 100644 --- a/tests/e2e_builtin_tool_coverage.rs +++ b/tests/e2e_builtin_tool_coverage.rs @@ -27,6 +27,8 @@ mod tests { let rig = TestRigBuilder::new() .with_trace(trace.clone()) + .with_auto_approve_tools(true) + .with_skills() .build() .await; @@ -60,6 +62,8 @@ mod tests { let rig = TestRigBuilder::new() .with_trace(trace.clone()) + .with_auto_approve_tools(true) + .with_skills() .build() .await; @@ -97,6 +101,8 @@ mod tests { let rig = TestRigBuilder::new() .with_trace(trace.clone()) + .with_auto_approve_tools(true) + .with_skills() .build() .await; @@ -197,7 +203,114 @@ mod tests { } // ----------------------------------------------------------------------- - // Test 6: job_create_status + // Test 6: routine_system_event_emit + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn routine_system_event_emit() { + let trace = LlmTrace::from_file(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/llm_traces/tools/routine_system_event_emit.json" + )) + .expect("failed to load routine_system_event_emit.json"); + + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .with_auto_approve_tools(true) + .build() + .await; + + rig.send_message("Create a system-event routine and emit an event") + .await; + let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await; + + rig.verify_trace_expects(&trace, &responses); + + let completed = rig.tool_calls_completed(); + assert!( + completed.iter().any(|(n, ok)| n == "event_emit" && *ok), + "event_emit should succeed: {completed:?}" + ); + + let results = rig.tool_results(); + let emit_result = results + .iter() + .find(|(n, _)| n == "event_emit") + .expect("event_emit result missing"); + assert!( + emit_result.1.contains("fired_routines"), + "event_emit should report fired routine count: {:?}", + emit_result.1 + ); + // Verify at least one routine actually fired (not just that the key exists). + let emit_json: serde_json::Value = + serde_json::from_str(&emit_result.1).expect("event_emit result should be valid JSON"); + assert!( + emit_json["fired_routines"].as_u64().unwrap_or(0) > 0, + "event_emit should have fired at least one routine: {:?}", + emit_result.1 + ); + + rig.shutdown(); + } + + // ----------------------------------------------------------------------- + // Test 7: skill_install_routine_webhook_sim + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn skill_install_routine_webhook_sim() { + let trace = LlmTrace::from_file(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/llm_traces/tools/skill_install_routine_webhook_sim.json" + )) + .expect("failed to load skill_install_routine_webhook_sim.json"); + + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .with_skills() + .with_auto_approve_tools(true) + .build() + .await; + + rig.send_message("Install the workflow skill template and simulate a webhook routine run") + .await; + let responses = rig.wait_for_responses(1, Duration::from_secs(20)).await; + rig.verify_trace_expects(&trace, &responses); + + let completed = rig.tool_calls_completed(); + assert!( + completed.iter().any(|(n, _)| n == "skill_install"), + "skill_install should be called: {completed:?}" + ); + for tool in &["routine_create", "event_emit", "routine_history"] { + assert!( + completed.iter().any(|(n, ok)| n == tool && *ok), + "{tool} should succeed: {completed:?}" + ); + } + + let results = rig.tool_results(); + let emit_result = results + .iter() + .find(|(n, _)| n == "event_emit") + .expect("event_emit result missing"); + assert!( + emit_result.1.contains("fired_routines"), + "event_emit should include fired_routines: {:?}", + emit_result.1 + ); + + let _history_result = results + .iter() + .find(|(n, _)| n == "routine_history") + .expect("routine_history result missing"); + + rig.shutdown(); + } + + // ----------------------------------------------------------------------- + // Test 8: job_create_status // ----------------------------------------------------------------------- // Uses {{call_cj_1.job_id}} template to forward the dynamic UUID from // create_job's result into job_status's arguments. @@ -266,7 +379,7 @@ mod tests { } // ----------------------------------------------------------------------- - // Test 7: job_list_cancel + // Test 9: job_list_cancel // ----------------------------------------------------------------------- // Uses {{call_cj_lc.job_id}} template to forward the dynamic UUID from // create_job into cancel_job. diff --git a/tests/e2e_routine_heartbeat.rs b/tests/e2e_routine_heartbeat.rs index 4d26e5da..a9ef086b 100644 --- a/tests/e2e_routine_heartbeat.rs +++ b/tests/e2e_routine_heartbeat.rs @@ -255,7 +255,151 @@ mod tests { } // ----------------------------------------------------------------------- - // Test 3: routine_cooldown + // Test 3: system_event_trigger_matches_and_filters + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn system_event_trigger_matches_and_filters() { + let (db, _tmp) = create_test_db().await; + let ws = create_workspace(&db); + + let trace = LlmTrace::single_turn( + "test-system-event-match", + "event", + vec![TraceStep { + request_hint: None, + response: TraceResponse::Text { + content: "System event handled".to_string(), + input_tokens: 40, + output_tokens: 8, + }, + expected_tool_results: vec![], + }], + ); + let llm = Arc::new(TraceLlm::from_trace(trace)); + let (notify_tx, _notify_rx) = tokio::sync::mpsc::channel(16); + + // Create minimal ToolRegistry and SafetyLayer for test. + let tools = Arc::new(ToolRegistry::new()); + let safety_config = SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: true, + }; + let safety = Arc::new(SafetyLayer::new(&safety_config)); + + let engine = Arc::new(RoutineEngine::new( + RoutineConfig::default(), + db.clone(), + llm, + ws, + notify_tx, + None, + tools, + safety, + )); + + let mut filters = std::collections::HashMap::new(); + filters.insert("repository".to_string(), "nearai/ironclaw".to_string()); + + let routine = make_routine( + "github-issue-opened", + Trigger::SystemEvent { + source: "github".to_string(), + event_type: "issue.opened".to_string(), + filters, + }, + "Summarize the issue and propose an implementation plan.", + ); + db.create_routine(&routine).await.expect("create_routine"); + engine.refresh_event_cache().await; + + // Matching event should fire. + let fired = engine + .emit_system_event( + "github", + "issue.opened", + &serde_json::json!({ + "repository": "nearai/ironclaw", + "issue_number": 42 + }), + Some("default"), + ) + .await; + assert_eq!(fired, 1, "Expected one routine to fire for matching event"); + + tokio::time::sleep(Duration::from_millis(300)).await; + + let runs = db + .list_routine_runs(routine.id, 10) + .await + .expect("list runs"); + assert!( + !runs.is_empty(), + "Expected run history after matching event" + ); + + // Wrong event type should not fire. + let fired_wrong_type = engine + .emit_system_event( + "github", + "issue.closed", + &serde_json::json!({"repository": "nearai/ironclaw"}), + Some("default"), + ) + .await; + assert_eq!( + fired_wrong_type, 0, + "Expected no routine for wrong event type" + ); + + // Wrong filter value should not fire. + let fired_wrong_filter = engine + .emit_system_event( + "github", + "issue.opened", + &serde_json::json!({"repository": "other/repo"}), + Some("default"), + ) + .await; + assert_eq!( + fired_wrong_filter, 0, + "Expected no routine for filter mismatch" + ); + + // Case-insensitive source/event_type should still match. + let fired_case = engine + .emit_system_event( + "GitHub", + "Issue.Opened", + &serde_json::json!({ + "repository": "nearai/ironclaw", + "issue_number": 99 + }), + Some("default"), + ) + .await; + assert_eq!( + fired_case, 1, + "Expected case-insensitive match on source/event_type" + ); + + // Case-insensitive filter values should match. + let fired_filter_case = engine + .emit_system_event( + "github", + "issue.opened", + &serde_json::json!({"repository": "NearAI/IronClaw"}), + Some("default"), + ) + .await; + assert_eq!( + fired_filter_case, 1, + "Expected case-insensitive match on filter values" + ); + } + + // ----------------------------------------------------------------------- + // Test 4: routine_cooldown // ----------------------------------------------------------------------- #[tokio::test] @@ -345,7 +489,7 @@ mod tests { } // ----------------------------------------------------------------------- - // Test 4: heartbeat_findings + // Test 5: heartbeat_findings // ----------------------------------------------------------------------- #[tokio::test] @@ -407,7 +551,7 @@ mod tests { } // ----------------------------------------------------------------------- - // Test 5: heartbeat_empty_skip + // Test 6: heartbeat_empty_skip // ----------------------------------------------------------------------- #[tokio::test] diff --git a/tests/fixtures/llm_traces/tools/routine_system_event_emit.json b/tests/fixtures/llm_traces/tools/routine_system_event_emit.json new file mode 100644 index 00000000..484574bb --- /dev/null +++ b/tests/fixtures/llm_traces/tools/routine_system_event_emit.json @@ -0,0 +1,64 @@ +{ + "model_name": "test-routine-system-event-emit", + "expects": { + "tools_used": ["routine_create", "event_emit"], + "all_tools_succeeded": true, + "tool_results_contain": { + "event_emit": "fired_routines" + } + }, + "steps": [ + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_rc_1", + "name": "routine_create", + "arguments": { + "name": "gh-issue-emit-test", + "description": "React to GitHub issue.opened events", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "issue.opened", + "action_type": "full_job", + "prompt": "Summarize the new issue and propose next steps." + } + } + ], + "input_tokens": 80, + "output_tokens": 30 + } + }, + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_ee_1", + "name": "event_emit", + "arguments": { + "event_source": "github", + "event_type": "issue.opened", + "payload": { + "repository": "nearai/ironclaw", + "issue_number": 123, + "title": "Support event-driven project workflow" + } + } + } + ], + "input_tokens": 140, + "output_tokens": 28 + } + }, + { + "response": { + "type": "text", + "content": "Created a system-event routine and emitted a matching GitHub event. The routine fired successfully.", + "input_tokens": 200, + "output_tokens": 18 + } + } + ] +} diff --git a/tests/fixtures/llm_traces/tools/skill_install_routine_webhook_sim.json b/tests/fixtures/llm_traces/tools/skill_install_routine_webhook_sim.json new file mode 100644 index 00000000..ef36df3e --- /dev/null +++ b/tests/fixtures/llm_traces/tools/skill_install_routine_webhook_sim.json @@ -0,0 +1,100 @@ +{ + "model_name": "test-skill-install-routine-webhook-sim", + "expects": { + "tools_used": ["skill_install", "routine_create", "event_emit", "routine_history"], + "tool_results_contain": { + "event_emit": "fired_routines" + } + }, + "steps": [ + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_skill_install_1", + "name": "skill_install", + "arguments": { + "name": "wf-orchestrator-trace-install-1", + "content": "---\nname: wf-orchestrator-trace-install-1\ndescription: Minimal workflow skill for trace install validation\nactivation:\n keywords: [\"workflow\", \"orchestrator\"]\n---\n\nYou are a minimal workflow skill used for trace install validation.\n" + } + } + ], + "input_tokens": 120, + "output_tokens": 32 + } + }, + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_routine_create_1", + "name": "routine_create", + "arguments": { + "name": "wf-webhook-sim-trace", + "description": "Trace routine to simulate webhook event flow", + "trigger_type": "system_event", + "event_source": "github", + "event_type": "issue.opened", + "event_filters": { + "repository": "nearai/ironclaw" + }, + "action_type": "full_job", + "prompt": "When issue webhook event arrives, start implementation loop and create branch/PR updates." + } + } + ], + "input_tokens": 170, + "output_tokens": 36 + } + }, + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_event_emit_1", + "name": "event_emit", + "arguments": { + "event_source": "github", + "event_type": "issue.opened", + "payload": { + "repository": "nearai/ironclaw", + "issue_number": 4242, + "sender": "trace-bot" + } + } + } + ], + "input_tokens": 210, + "output_tokens": 28 + } + }, + { + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_routine_history_1", + "name": "routine_history", + "arguments": { + "name": "wf-webhook-sim-trace", + "limit": 5 + } + } + ], + "input_tokens": 240, + "output_tokens": 22 + } + }, + { + "response": { + "type": "text", + "content": "Installed the skill template, created a system-event routine, emitted a webhook-equivalent event, and verified the routine run history.", + "input_tokens": 280, + "output_tokens": 25 + } + } + ] +} diff --git a/tests/sighup_reload_integration.rs b/tests/sighup_reload_integration.rs new file mode 100644 index 00000000..3e009ade --- /dev/null +++ b/tests/sighup_reload_integration.rs @@ -0,0 +1,170 @@ +//! Integration test for SIGHUP hot-reload of HTTP webhook configuration. +//! +//! This test verifies that: +//! 1. SIGHUP triggers config reload from DB/environment +//! 2. Address changes cause listener restart +//! 3. Secret changes take effect immediately (zero-downtime) +//! 4. Old listener is shut down after successful restart + +#![cfg(unix)] + +use std::time::Duration; + +#[tokio::test] +#[ignore] // Requires full ironclaw binary and database setup +async fn test_sighup_config_reload_address_change() { + // This is a placeholder integration test structure. + // It demonstrates the test approach and can be run against a live ironclaw instance. + // + // To run this test manually: + // 1. Start ironclaw with HTTP_PORT=19000 HTTP_WEBHOOK_SECRET=initial-secret + // 2. Run: cargo test --test sighup_reload_integration -- --ignored --nocapture + // + // The test will: + // - Verify initial webhook responds on port 19000 with "initial-secret" + // - Update environment/DB to use port 19001 and "new-secret" + // - Send SIGHUP to ironclaw + // - Verify old port 19000 stops responding + // - Verify new port 19001 responds with "new-secret" + + let initial_port = 19000u16; + let _new_port = 19001u16; + let initial_secret = "initial-secret"; + let _new_secret = "new-secret"; + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("Failed to build HTTP client"); + + // Verify initial webhook is listening + let initial_addr = format!("http://127.0.0.1:{}/webhook", initial_port); + let response = client + .post(&initial_addr) + .json(&serde_json::json!({ + "content": "test", + "secret": initial_secret + })) + .send() + .await; + + assert!( + response.is_ok(), + "Initial webhook should be listening on port {}", + initial_port + ); + assert_eq!( + response.unwrap().status(), + 200, + "Request with correct secret should succeed" + ); + + // In a real test, we would: + // 1. Update the database or environment variables for the new config + // 2. Send SIGHUP to the ironclaw process + // 3. Wait for reload to complete + // 4. Verify new listener is active and old one is inactive + // 5. Verify secret change took effect + + println!("SIGHUP reload test structure is in place."); + println!("This test requires a running ironclaw instance to verify actual behavior."); +} + +#[tokio::test] +#[ignore] // Requires full ironclaw binary +async fn test_sighup_secret_update_zero_downtime() { + // Test that secret changes take effect immediately without restarting the listener. + // + // Setup: + // - Start ironclaw with HTTP_PORT=19002 HTTP_WEBHOOK_SECRET=original-secret + // + // Test flow: + // 1. Make request with "original-secret" → 200 OK + // 2. Update DB secret to "updated-secret" + // 3. Send SIGHUP + // 4. Make request with "original-secret" → 401 Unauthorized + // 5. Make request with "updated-secret" → 200 OK + // 6. Verify listener is still on same port (no restart) + + let port = 19002u16; + let original_secret = "original-secret"; + let _updated_secret = "updated-secret"; + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("Failed to build HTTP client"); + + let webhook_url = format!("http://127.0.0.1:{}/webhook", port); + + // Verify original secret works + let response = client + .post(&webhook_url) + .json(&serde_json::json!({ + "content": "test", + "secret": original_secret + })) + .send() + .await; + + assert!( + response.is_ok(), + "Initial request with correct secret should succeed" + ); + assert_eq!(response.unwrap().status(), 200); + + // After SIGHUP with updated secret: + // - Original secret should fail + // - Updated secret should succeed + // (This is verified by the hot-swap unit test; integration test + // structure is in place for end-to-end verification) + + println!("Zero-downtime secret update test structure is in place."); +} + +#[tokio::test] +#[ignore] // Requires manual setup +async fn test_sighup_rollback_on_address_bind_failure() { + // Test that if restart_with_addr fails, the old listener remains active + // and state is restored. + // + // Setup: + // - Start ironclaw with HTTP_PORT=19003 HTTP_WEBHOOK_SECRET=test-secret + // + // Test flow: + // 1. Make request to port 19003 → 200 OK + // 2. Update DB to use invalid address (e.g., port 1, which requires root) + // 3. Send SIGHUP + // 4. Verify old listener on port 19003 is still responding + // 5. Verify state was restored (config still shows port 19003) + + let original_port = 19003u16; + let secret = "test-secret"; + + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .expect("Failed to build HTTP client"); + + let webhook_url = format!("http://127.0.0.1:{}/webhook", original_port); + + // Verify original listener is working + let response = client + .post(&webhook_url) + .json(&serde_json::json!({ + "content": "test", + "secret": secret + })) + .send() + .await; + + assert!(response.is_ok(), "Original listener should be responding"); + assert_eq!(response.unwrap().status(), 200); + + // After SIGHUP with invalid address: + // - Original listener should still respond + // - No downtime should have occurred + // (Verified by webhook_server unit test; integration structure in place) + + println!("SIGHUP rollback test structure is in place."); +} diff --git a/tests/support/test_rig.rs b/tests/support/test_rig.rs index bedc6d4a..cc7d77ac 100644 --- a/tests/support/test_rig.rs +++ b/tests/support/test_rig.rs @@ -379,6 +379,8 @@ pub struct TestRigBuilder { llm: Option>, max_tool_iterations: usize, injection_check: bool, + auto_approve_tools: Option, + enable_skills: bool, enable_routines: bool, http_exchanges: Vec, extra_tools: Vec>, @@ -392,6 +394,8 @@ impl TestRigBuilder { llm: None, max_tool_iterations: 10, injection_check: false, + auto_approve_tools: None, + enable_skills: false, enable_routines: false, http_exchanges: Vec::new(), extra_tools: Vec::new(), @@ -432,6 +436,18 @@ impl TestRigBuilder { self } + /// Override agent-level automatic approval of `UnlessAutoApproved` tools. + pub fn with_auto_approve_tools(mut self, enable: bool) -> Self { + self.auto_approve_tools = Some(enable); + self + } + + /// Enable skill discovery and registration for this test rig. + pub fn with_skills(mut self) -> Self { + self.enable_skills = true; + self + } + /// Enable the routines system so the scheduler is wired with a `RoutineEngine`, /// allowing routine jobs to actually execute. Routine tools are always registered /// but require the engine to dispatch jobs. @@ -466,6 +482,8 @@ impl TestRigBuilder { llm, max_tool_iterations, injection_check, + auto_approve_tools, + enable_skills, enable_routines, http_exchanges: explicit_http_exchanges, extra_tools, @@ -491,6 +509,10 @@ impl TestRigBuilder { let mut config = Config::for_testing(db_path, skills_dir, installed_skills_dir); config.agent.max_tool_iterations = max_tool_iterations; config.safety.injection_check_enabled = injection_check; + config.skills.enabled = enable_skills; + if let Some(v) = auto_approve_tools { + config.agent.auto_approve_tools = v; + } // 3. Create SessionManager + LogBroadcaster. let session = Arc::new(SessionManager::new(SessionConfig::default())); @@ -540,7 +562,7 @@ impl TestRigBuilder { ); builder.with_database(Arc::clone(&db)); builder.with_llm(llm); - let components = builder + let mut components = builder .build_all() .await .expect("AppBuilder::build_all() failed in test rig"); @@ -583,6 +605,21 @@ impl TestRigBuilder { .register_routine_tools(Arc::clone(db_arc), engine); } + // Skills tools: ensure tests use temp skill dirs (sandbox-safe) even if + // AppBuilder did not wire them for this environment. + if enable_skills { + let registry = Arc::new(std::sync::RwLock::new( + ironclaw::skills::SkillRegistry::new(temp_dir.path().join("skills")) + .with_installed_dir(temp_dir.path().join("installed_skills")), + )); + let catalog = ironclaw::skills::catalog::shared_catalog(); + components + .tools + .register_skill_tools(Arc::clone(®istry), Arc::clone(&catalog)); + components.skill_registry = Some(registry); + components.skill_catalog = Some(catalog); + } + // Register any extra test-specific tools. for tool in extra_tools { components.tools.register(tool).await; diff --git a/tests/support/trace_llm.rs b/tests/support/trace_llm.rs index e09ee9d9..ba3e5744 100644 --- a/tests/support/trace_llm.rs +++ b/tests/support/trace_llm.rs @@ -429,7 +429,7 @@ impl TraceLlm { } /// Strip `...\n` - /// wrapper and unescape XML entities from safety-layer output. + /// wrapper from safety-layer output. fn unwrap_tool_output(content: &str) -> std::borrow::Cow<'_, str> { let trimmed = content.trim(); if let Some(rest) = trimmed.strip_prefix("") { let body = inner[..close].trim(); - // Reverse XML escaping applied by safety layer. - if body.contains("&") || body.contains("<") || body.contains(">") { - return std::borrow::Cow::Owned( - body.replace("&", "&") - .replace("<", "<") - .replace(">", ">"), - ); - } return std::borrow::Cow::Borrowed(body); } } diff --git a/tests/ws_gateway_integration.rs b/tests/ws_gateway_integration.rs index da44f766..6f66e19e 100644 --- a/tests/ws_gateway_integration.rs +++ b/tests/ws_gateway_integration.rs @@ -340,3 +340,72 @@ async fn test_ws_multiple_events_in_sequence() { ws.close(None).await.unwrap(); } + +/// Regression test: verify session lock is not held during API handler operations. +/// +/// This test ensures that concurrent API requests (e.g., listing threads) don't +/// block the agent loop from processing messages. Previously, chat_threads_handler +/// and chat_history_handler held session locks during slow DB operations, which +/// would deadlock the agent loop waiting to resolve sessions for incoming messages. +/// +/// The test verifies that concurrent access to session state completes quickly +/// without deadlock. If locks are heavily contended, the test will timeout. +#[tokio::test] +async fn test_session_lock_not_held_during_api_operations() { + use ironclaw::agent::SessionManager; + + let (_addr, _state, _agent_rx) = start_test_server().await; + + // Create a session manager and attach it to state + let session_manager = Arc::new(SessionManager::new()); + + // Note: We can't directly modify state.session_manager in the test due to its type. + // Instead, we test the session manager directly in isolation to verify lock behavior. + + // Spawn concurrent operations simulating API handler + agent loop interaction + let mut handles = vec![]; + + // Simulate API handler threads accessing sessions + for user_id in 0..5 { + let sm = session_manager.clone(); + handles.push(tokio::spawn(async move { + for _ in 0..20 { + let session = sm.get_or_create_session(&format!("user-{}", user_id)).await; + // Lock and release quickly (simulating API reading session state) + { + let _sess = session.lock().await; + tokio::time::sleep(Duration::from_micros(100)).await; + } + } + })); + } + + // Simulate agent loop thread resolving threads + let sm = session_manager.clone(); + let agent_handle = tokio::spawn(async move { + for i in 0..20 { + let (_session, _thread_id) = sm + .resolve_thread(&format!("user-{}", i % 5), "gateway", None) + .await; + // Should not block waiting for API handler locks + tokio::time::sleep(Duration::from_micros(100)).await; + } + }); + handles.push(agent_handle); + + // Wait for all tasks to complete within reasonable time + // If session locks are held during slow operations, this will timeout + let timeout_duration = Duration::from_secs(5); + let wait_result = timeout(timeout_duration, async { + for handle in handles { + let _ = handle.await; + } + }) + .await; + + assert!( + wait_result.is_ok(), + "Concurrent session access deadlocked or timed out. \ + This suggests session locks are held too long during I/O operations." + ); +}