diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..32b9468c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +target/ +.git/ +.env +.env.* +*.md +!CLAUDE.md +node_modules/ +tools-src/ diff --git a/.env.example b/.env.example index 1a80fc02..e3b629ee 100644 --- a/.env.example +++ b/.env.example @@ -1,15 +1,15 @@ # Database Configuration -DATABASE_URL=postgres://ironclaw:password@localhost:5432/ironclaw +DATABASE_URL=postgres://localhost/ironclaw DATABASE_POOL_SIZE=10 # LLM Provider (NEAR AI) # NEAR AI provides a unified interface to all models with user authentication -# Session token is stored in ~/.near-agent/session.json and managed automatically. +# Session token is stored in ~/.ironclaw/session.json and managed automatically. # On first run, the agent will open a browser for OAuth authentication. NEARAI_MODEL=claude-3-5-sonnet-20241022 NEARAI_BASE_URL=https://cloud-api.near.ai NEARAI_AUTH_URL=https://private.near.ai -# NEARAI_SESSION_PATH=~/.near-agent/session.json # optional, default shown +# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown # Channel Configuration # CLI is always enabled diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml new file mode 100644 index 00000000..19f7d725 --- /dev/null +++ b/.github/workflows/code_style.yml @@ -0,0 +1,22 @@ +name: Code Style +on: + pull_request: + +jobs: + codestyle: + name: Code Style (fmt + clippy) + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + profile: minimal + components: rustfmt, clippy + - uses: Swatinem/rust-cache@v2 + - name: Check formatting + run: | + cargo fmt --all -- --check + - name: Check lints (cargo clippy) + run: cargo clippy -- -D warnings diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml new file mode 100644 index 00000000..142b2b20 --- /dev/null +++ b/.github/workflows/release-plz.yml @@ -0,0 +1,67 @@ +name: Release-plz + +on: + push: + branches: + - main + +jobs: + + # Release unpublished packages. + release-plz-release: + if: ${{ github.repository_owner == 'nearai' }} + name: Release-plz release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - &checkout + name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - &install-rust + name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + # Generating a GitHub token, so that PRs and tags created by + # the release-plz-action can trigger actions workflows. + - name: Generate GitHub token + uses: actions/create-github-app-token@v2 + id: generate-token + with: + # GitHub App ID secret name + app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }} + # GitHub App private key secret name + private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }} + - name: Run release-plz + uses: release-plz/action@v0.5 + with: + command: release + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + # Create a PR with the new versions and changelog, preparing the next release. + release-plz-pr: + if: ${{ github.repository_owner == 'nearai' }} + name: Release-plz PR + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + concurrency: + group: release-plz-${{ github.ref }} + cancel-in-progress: false + steps: + - *checkout + - *install-rust + - uses: Swatinem/rust-cache@v2 + - name: Run release-plz + uses: release-plz/action@v0.5 + with: + command: release-pr + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..530a221a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,300 @@ +# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist +# +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that the GitHub Release will be created with a generated +# title/body based on your changelogs. + +name: Release +permissions: + "contents": "write" + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + pull_request: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + +jobs: + # Run 'dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.3/cargo-dist-installer.sh | sh" + - name: Cache dist + uses: actions/upload-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/dist + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - uses: swatinem/rust-cache@v2 + with: + key: ${{ join(matrix.targets, '-') }} + cache-provider: ${{ matrix.cache_provider }} + - name: Install dist + run: ${{ matrix.install_dist.run }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v4 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + # Create a GitHub Release while uploading all files to it + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}" + ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + # Write and read notes from a file to avoid quoting breaking things + echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt + + gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..13fc8410 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,21 @@ +name: Run Tests +on: + pull_request: + push: + branches: + - main + +jobs: + tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + profile: minimal + - uses: Swatinem/rust-cache@v2 + - name: Run Tests + run: cargo test --all-features -- --nocapture diff --git a/.gitignore b/.gitignore index e9846352..0f80f04c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ target/ +# WASM build artifacts (loaded from disk, not bundled) +*.wasm + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..f04b7e6a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,117 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.3](https://github.com/nearai/ironclaw/compare/v0.1.2...v0.1.3) - 2026-02-12 + +### Other + +- Enabled builds caching during CI/CD +- Disabled npm publishing as the name is already taken + +## [0.1.2](https://github.com/nearai/ironclaw/compare/v0.1.1...v0.1.2) - 2026-02-12 + +### Other + +- Added Installation instructions for the pre-built binaries +- Disabled Windows ARM64 builds as auto-updater [provided by cargo-dist] does not support this platform yet and it is not a common platform for us to support + +## [0.1.1](https://github.com/nearai/ironclaw/compare/v0.1.0...v0.1.1) - 2026-02-12 + +### Other + +- Renamed the secrets in release-plz.yml to match the configuration +- Make sure that the binaries release CD it kicking in after release-plz + +## [0.1.0](https://github.com/nearai/ironclaw/releases/tag/v0.1.0) - 2026-02-12 + +### Added + +- Add multi-provider LLM support via rig-core adapter ([#36](https://github.com/nearai/ironclaw/pull/36)) +- Sandbox jobs ([#4](https://github.com/nearai/ironclaw/pull/4)) +- Add Google Suite & Telegram WASM tools ([#9](https://github.com/nearai/ironclaw/pull/9)) +- Improve CLI ([#5](https://github.com/nearai/ironclaw/pull/5)) + +### Fixed + +- resolve runtime panic in Linux keychain integration ([#32](https://github.com/nearai/ironclaw/pull/32)) + +### Other + +- Skip release-plz on forks +- Upgraded release-plz CD pipeline +- Added CI/CD and release pipelines ([#45](https://github.com/nearai/ironclaw/pull/45)) +- DM pairing + Telegram channel improvements ([#17](https://github.com/nearai/ironclaw/pull/17)) +- Fixes build, adds missing sse event and correct command ([#11](https://github.com/nearai/ironclaw/pull/11)) +- Codex/feature parity pr hook ([#6](https://github.com/nearai/ironclaw/pull/6)) +- Add WebSocket gateway and control plane ([#8](https://github.com/nearai/ironclaw/pull/8)) +- select bundled Telegram channel and auto-install ([#3](https://github.com/nearai/ironclaw/pull/3)) +- Adding skills for reusable work +- Fix MCP tool calls, approval loop, shutdown, and improve web UI +- Add auth mode, fix MCP token handling, and parallelize startup loading +- Merge remote-tracking branch 'origin/main' into ui +- Adding web UI +- Rename `setup` CLI command to `onboard` for compatibility +- Add in-chat extension discovery, auth, and activation system +- Add Telegram typing indicator via WIT on-status callback +- Add proactivity features: memory CLI, session pruning, self-repair notifications, slash commands, status diagnostics, context warnings +- Add hosted MCP server support with OAuth 2.1 and token refresh +- Add interactive setup wizard and persistent settings +- Rebrand to IronClaw with security-first mission +- Fix build_software tool stuck in planning mode loop +- Enable sandbox by default +- Fix Telegram Markdown formatting and clarify tool/memory distinctions +- Simplify Telegram channel config with host-injected tunnel/webhook settings +- Apply Telegram channel learnings to WhatsApp implementation +- Merge remote-tracking branch 'origin/main' +- Docker file for sandbox +- Replace hardcoded intent patterns with job tools +- Fix router test to match intentional job creation patterns +- Add Docker execution sandbox for secure shell command isolation +- Move setup wizard credentials to database storage +- Add interactive setup wizard for first-run configuration +- Add Telegram Bot API channel as WASM module +- Add OpenClaw feature parity tracking matrix +- Add Chat Completions API support and expand REPL debugging +- Implementing channels to be handled in wasm +- Support non interactive mode and model selection +- Implement tool approval, fix tool definition refresh, and wire embeddings +- Tool use +- Wiring more +- Add heartbeat integration, planning phase, and auto-repair +- Login flow +- Extend support for session management +- Adding builder capability +- Load tools at launch +- Fix multiline message rendering in TUI +- Parse NEAR AI alternative response format with output field +- Handle NEAR AI plain text responses +- Disable mouse capture to allow text selection in TUI +- Add verbose logging to debug empty NEAR AI responses +- Improve NEAR AI response parsing for varying response formats +- Show status/thinking messages in chat window, debug empty responses +- Add timeout and logging to NEAR AI provider +- Add status updates to show agent thinking/processing state +- Add CLI subcommands for WASM tool management +- Fix TUI shutdown: send /shutdown message and handle in agent loop +- Remove SimpleCliChannel, add Ctrl+D twice quit, redirect logs to TUI +- Fix TuiChannel integration and enable in main.rs +- Integrate Codex patterns: task scheduler, TUI, sessions, compaction +- Adding LICENSE +- Add README with IronClaw branding +- Add WASM sandbox secure API extension +- Wire database Store into agent loop +- Implementing WASM runtime +- Add workspace integration tests +- Compact memory_tree output format +- Replace memory_list with memory_tree tool +- Simplify workspace to path-based storage, remove legacy code +- Add NEAR AI chat-api as default LLM provider +- Add CLAUDE.md project documentation +- Add workspace and memory system (OpenClaw-inspired) +- Initial implementation of the agent framework diff --git a/CLAUDE.md b/CLAUDE.md index 2064847a..5664469b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,8 +11,13 @@ - **Always available** - Multi-channel access with proactive background execution ### Features -- **Multi-channel input**: TUI (Ratatui), HTTP webhooks, Telegram, WhatsApp, Slack (WASM channels) +- **Multi-channel input**: TUI (Ratatui), HTTP webhooks, WASM channels (Telegram, Slack), web gateway - **Parallel job execution** with state machine and self-repair for stuck jobs +- **Sandbox execution**: Docker container isolation with orchestrator/worker pattern +- **Claude Code mode**: Delegate jobs to Claude CLI inside containers +- **Routines**: Scheduled (cron) and reactive (event, webhook) task execution +- **Web gateway**: Browser UI with SSE/WebSocket real-time streaming +- **Extension management**: Install, auth, activate MCP/WASM extensions - **Extensible tools**: Built-in tools, WASM sandbox, MCP client, dynamic builder - **Persistent memory**: Workspace with hybrid search (FTS + vector via RRF) - **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection @@ -59,7 +64,9 @@ src/ │ ├── context_monitor.rs # Memory pressure detection │ ├── undo.rs # Turn-based undo/redo with checkpoints │ ├── submission.rs # Submission parsing (undo, redo, compact, clear, etc.) -│ └── task.rs # Sub-task execution framework +│ ├── task.rs # Sub-task execution framework +│ ├── routine.rs # Routine types (Trigger, Action, Guardrails) +│ └── routine_engine.rs # Routine execution (cron ticker, event matcher) │ ├── channels/ # Multi-channel input │ ├── channel.rs # Channel trait, IncomingMessage, OutgoingResponse @@ -72,8 +79,33 @@ src/ │ │ ├── overlay.rs # Approval overlays │ │ └── composer.rs # Message composition │ ├── http.rs # HTTP webhook (axum) with secret validation -│ ├── slack.rs # Stub -│ └── telegram.rs # Stub +│ ├── repl.rs # Simple REPL (for testing) +│ ├── web/ # Web gateway (browser UI) +│ │ ├── mod.rs # Gateway builder, startup +│ │ ├── server.rs # Axum router, 40+ API endpoints +│ │ ├── sse.rs # SSE broadcast manager +│ │ ├── ws.rs # WebSocket gateway + connection tracking +│ │ ├── types.rs # Request/response types, SseEvent enum +│ │ ├── auth.rs # Bearer token auth middleware +│ │ ├── log_layer.rs # Tracing layer for log streaming +│ │ └── static/ # HTML, CSS, JS (single-page app) +│ └── wasm/ # WASM channel runtime +│ ├── mod.rs +│ ├── bundled.rs # Bundled channel discovery +│ └── wrapper.rs # Channel trait wrapper for WASM modules +│ +├── orchestrator/ # Internal HTTP API for sandbox containers +│ ├── mod.rs +│ ├── api.rs # Axum endpoints (LLM proxy, events, prompts) +│ ├── auth.rs # Per-job bearer token store +│ └── job_manager.rs # Container lifecycle (create, stop, cleanup) +│ +├── worker/ # Runs inside Docker containers +│ ├── mod.rs +│ ├── runtime.rs # Worker execution loop (tool calls, LLM) +│ ├── claude_bridge.rs # Claude Code bridge (spawns claude CLI) +│ ├── api.rs # HTTP client to orchestrator +│ └── proxy_llm.rs # LlmProvider that proxies through orchestrator │ ├── safety/ # Prompt injection defense │ ├── sanitizer.rs # Pattern detection, content escaping @@ -96,6 +128,9 @@ src/ │ │ ├── file.rs # ReadFile, WriteFile, ListDir, ApplyPatch │ │ ├── shell.rs # Shell command execution │ │ ├── memory.rs # Memory tools (search, write, read, tree) +│ │ ├── job.rs # CreateJob, ListJobs, JobStatus, CancelJob +│ │ ├── routine.rs # routine_create/list/update/delete/history +│ │ ├── extension_tools.rs # Extension install/auth/activate/remove │ │ └── marketplace.rs, ecommerce.rs, taskrabbit.rs, restaurant.rs (stubs) │ ├── builder/ # Dynamic tool building │ │ ├── core.rs # BuildRequirement, SoftwareType, Language @@ -116,6 +151,12 @@ src/ │ ├── rate_limiter.rs # Per-tool rate limiting │ └── storage.rs # Linear memory persistence │ +├── db/ # Database abstraction layer +│ ├── mod.rs # Database trait (~60 async methods) +│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository) +│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite) +│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent) +│ ├── workspace/ # Persistent memory system (OpenClaw-inspired) │ ├── mod.rs # Workspace struct, memory operations │ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry @@ -157,8 +198,9 @@ When designing new features or systems, always prefer generic/extensible archite ### Error Handling - Use `thiserror` for error types in `error.rs` -- Never use `.unwrap()` in production code (tests are fine) +- Never use `.unwrap()` or `.expect()` in production code (tests are fine) - Map errors with context: `.map_err(|e| SomeError::Variant { reason: e.to_string() })?` +- Before committing, grep for `.unwrap()` and `.expect(` in changed files to catch violations mechanically ### Async - All I/O is async with tokio @@ -166,6 +208,7 @@ When designing new features or systems, always prefer generic/extensible archite - Use `RwLock` for concurrent read/write access ### Traits for Extensibility +- `Database` - Add new database backends (must implement all ~60 methods) - `Channel` - Add new input sources - `Tool` - Add new capabilities - `LlmProvider` - Add new LLM backends @@ -213,7 +256,12 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted Environment variables (see `.env.example`): ```bash +# Database backend (default: postgres) +DATABASE_BACKEND=postgres # or "libsql" / "turso" DATABASE_URL=postgres://user:pass@localhost/ironclaw +LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default) +# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional) +# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL # NEAR AI (required) NEARAI_SESSION_TOKEN=sess_... @@ -236,6 +284,30 @@ HEARTBEAT_ENABLED=true HEARTBEAT_INTERVAL_SECS=1800 # 30 minutes HEARTBEAT_NOTIFY_CHANNEL=tui HEARTBEAT_NOTIFY_USER=default + +# Web gateway +GATEWAY_ENABLED=true +GATEWAY_HOST=127.0.0.1 +GATEWAY_PORT=3001 +GATEWAY_AUTH_TOKEN=changeme # Required for API access +GATEWAY_USER_ID=default + +# Docker sandbox +SANDBOX_ENABLED=true +SANDBOX_IMAGE=ironclaw-worker:latest +SANDBOX_MEMORY_LIMIT_MB=512 +SANDBOX_TIMEOUT_SECS=1800 + +# Claude Code mode (runs inside sandbox containers) +CLAUDE_CODE_ENABLED=false +CLAUDE_CODE_MODEL=claude-sonnet-4-20250514 +CLAUDE_CODE_MAX_TURNS=50 +CLAUDE_CODE_CONFIG_DIR=/home/worker/.claude + +# Routines (scheduled/reactive execution) +ROUTINES_ENABLED=true +ROUTINES_CRON_INTERVAL=60 # Tick interval in seconds +ROUTINES_MAX_CONCURRENT=3 ``` ### NEAR AI Provider @@ -249,7 +321,51 @@ Session tokens have the format `sess_xxx` (37 characters). They are authenticate ## Database -Single migration in `migrations/V1__initial.sql`. Tables: +IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable. + +**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL). + +### Backends + +| Backend | Feature Flag | Default | Use Case | +|---------|-------------|---------|----------| +| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments | +| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud | + +```bash +# Build with PostgreSQL only (default) +cargo build + +# Build with libSQL only +cargo build --no-default-features --features libsql + +# Build with both backends available +cargo build --features "postgres,libsql" +``` + +### Database Trait + +The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence: +- Conversations, messages, metadata +- Jobs, actions, LLM calls, estimation snapshots +- Sandbox jobs, job events +- Routines, routine runs +- Tool failures, settings +- Workspace: documents, chunks, hybrid search + +Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL. + +### Schema + +**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`. + +**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types: +- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT` +- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx` +- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers +- PL/pgSQL functions -> SQLite triggers + +**Tables (both backends):** **Core:** - `conversations` - Multi-channel conversation tracking @@ -261,12 +377,41 @@ Single migration in `migrations/V1__initial.sql`. Tables: **Workspace/Memory:** - `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md") -- `memory_chunks` - Chunked content with FTS (tsvector) and vector (pgvector) indexes +- `memory_chunks` - Chunked content with FTS and vector indexes - `heartbeat_state` - Periodic execution tracking -Requires pgvector extension: `CREATE EXTENSION IF NOT EXISTS vector;` +**Other:** +- `routines`, `routine_runs` - Scheduled/reactive execution +- `settings` - Per-user key-value settings +- `tool_failures` - Self-repair tracking +- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure -Run migrations: `refinery migrate -c refinery.toml` +### Configuration + +```bash +# Backend selection (default: postgres) +DATABASE_BACKEND=libsql + +# PostgreSQL +DATABASE_URL=postgres://user:pass@localhost/ironclaw + +# libSQL (embedded) +LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path + +# libSQL (Turso cloud sync) +LIBSQL_URL=libsql://your-db.turso.io +LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set +``` + +### Current Limitations (libSQL backend) + +- **Workspace/memory system** not yet wired through Database trait (requires Store migration) +- **Secrets store** not yet available (still requires PostgresSecretsStore) +- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented) +- **Settings reload from DB** skipped (Config::from_db requires Store) +- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet) +- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage. +- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields. ## Safety Layer @@ -297,13 +442,14 @@ Key test patterns: ## Current Limitations / TODOs -1. **Slack/Telegram channels** - Stubs only, need implementation -2. **Domain-specific tools** - `marketplace.rs`, `restaurant.rs`, `taskrabbit.rs`, `ecommerce.rs` return placeholder responses; need real API integrations -3. **Integration tests** - Need testcontainers setup for PostgreSQL -4. **MCP stdio transport** - Only HTTP transport implemented -5. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed) -6. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access -7. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools +1. **Domain-specific tools** - `marketplace.rs`, `restaurant.rs`, `taskrabbit.rs`, `ecommerce.rs` return placeholder responses; need real API integrations +2. **Integration tests** - Need testcontainers setup for PostgreSQL +3. **MCP stdio transport** - Only HTTP transport implemented +4. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed) +5. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access +6. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools +7. **Webhook trigger endpoint** - Routines webhook trigger not yet exposed in web gateway +8. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard ### Completed @@ -320,6 +466,14 @@ Key test patterns: - ✅ **Tool approval enforcement** - Tools with `requires_approval()` (shell, http, file write/patch, build_software) now gate execution, track auto-approved tools per session - ✅ **Tool definition refresh** - Tool definitions refreshed each iteration so newly built tools become visible in same session - ✅ **Worker tool call handling** - Uses `respond_with_tools()` to properly execute tool calls when `select_tools()` returns empty +- ✅ **Gateway control plane** - Web gateway with 40+ API endpoints, SSE/WebSocket +- ✅ **Web Control UI** - Browser-based dashboard with chat, memory, jobs, logs, extensions, routines +- ✅ **Slack/Telegram channels** - Implemented as WASM tools +- ✅ **Docker sandbox** - Orchestrator/worker containers with per-job auth +- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers +- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails +- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI +- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode ## Adding a New Tool @@ -484,6 +638,37 @@ RUST_LOG=ironclaw=debug,tower_http=debug cargo run - Keep functions focused, extract helpers when logic is reused - Comments for non-obvious logic only +## Review & Fix Discipline + +Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback. + +### Fix the pattern, not just the instance +When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix. + +### Propagate architectural fixes to satellite types +If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase. + +### Schema translation is more than DDL +When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for: +- **Indexes** -- diff `CREATE INDEX` statements between the two schemas +- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`) +- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`) + +### Feature flag testing +When adding feature-gated code, test compilation with each feature in isolation: +```bash +cargo check # default features +cargo check --no-default-features --features libsql # libsql only +cargo check --all-features # all features +``` +Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature. + +### Mechanical verification before committing +Run these checks on changed files before committing: +- `grep -rnE '\.unwrap\(|\.expect\(' ` -- no panics in production +- `grep -rn 'super::' ` -- use `crate::` imports +- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/` + ## Workspace & Memory System Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure. @@ -558,7 +743,7 @@ Four tools for LLM use: ### Hybrid Search (RRF) -Combines full-text search (PostgreSQL `ts_rank_cd`) and vector similarity (pgvector cosine) using Reciprocal Rank Fusion: +Combines full-text search and vector similarity using Reciprocal Rank Fusion: ``` score(d) = Σ 1/(k + rank(d)) for each method where d appears @@ -566,6 +751,10 @@ score(d) = Σ 1/(k + rank(d)) for each method where d appears Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores. +**Backend differences:** +- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF +- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired) + ### Heartbeat System Proactive periodic execution (default: 30 minutes): diff --git a/Cargo.lock b/Cargo.lock index 02eac914..5e242b82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,7 +66,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.8.37", ] [[package]] @@ -182,6 +182,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -307,6 +313,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "async-task" version = "4.7.1" @@ -353,24 +381,52 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower 0.4.13", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ - "axum-core", + "axum-core 0.5.6", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -380,15 +436,32 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tokio-tungstenite 0.28.0", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.6" @@ -397,12 +470,12 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -420,6 +493,38 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.66.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.114", + "which 4.4.2", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -502,9 +607,9 @@ dependencies = [ "futures-util", "hex", "home", - "http", + "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-named-pipe", "hyper-rustls", "hyper-util", @@ -604,6 +709,9 @@ name = "bytes" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +dependencies = [ + "serde", +] [[package]] name = "cap-fs-ext" @@ -704,6 +812,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -739,8 +856,8 @@ dependencies = [ "tokio", "tracing", "url", - "which", - "windows-registry", + "which 8.0.0", + "windows-registry 0.5.3", ] [[package]] @@ -806,6 +923,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.56" @@ -903,6 +1031,16 @@ dependencies = [ "crossterm 0.29.0", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -974,7 +1112,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "regalloc2", - "rustc-hash", + "rustc-hash 2.1.1", "serde", "smallvec", "target-lexicon", @@ -1079,6 +1217,17 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "cron" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee8b2b4516038bc0f1d3c9934bcb4a13dd316e04abbc63c96757a6d75978532" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crossbeam" version = "0.8.4" @@ -1563,6 +1712,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1575,6 +1735,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -1621,6 +1787,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1641,6 +1822,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fs4" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.48.0", +] + [[package]] name = "funty" version = "2.0.0" @@ -1835,6 +2026,31 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.13" @@ -1846,7 +2062,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap 2.13.0", "slab", "tokio", @@ -1870,6 +2086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.12", + "allocator-api2", "serde", ] @@ -1889,6 +2106,15 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.4.1" @@ -1940,6 +2166,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1950,6 +2187,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1957,7 +2205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1968,11 +2216,17 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + [[package]] name = "httparse" version = "1.10.1" @@ -1985,6 +2239,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1995,9 +2273,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2015,7 +2293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -2029,15 +2307,43 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", - "webpki-roots", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.32", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] @@ -2050,17 +2356,19 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry 0.6.1", ] [[package]] @@ -2071,7 +2379,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -2283,13 +2591,13 @@ dependencies = [ [[package]] name = "ironclaw" -version = "0.1.0" +version = "0.1.3" dependencies = [ "aes-gcm", "aho-corasick", "anyhow", "async-trait", - "axum", + "axum 0.8.8", "base64 0.22.1", "blake3", "bollard", @@ -2297,15 +2605,19 @@ dependencies = [ "chromiumoxide", "chrono", "clap", + "cron", "crossterm 0.28.1", "deadpool-postgres", "dirs 6.0.0", "dotenvy", + "fs4", "futures", "hkdf", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", + "libsql", + "mime_guess", "open", "pgvector", "postgres-types", @@ -2314,15 +2626,17 @@ dependencies = [ "refinery", "regex", "reqwest", + "rig-core", "rust_decimal", "rust_decimal_macros", "rustyline", "secrecy", "secret-service", - "security-framework", + "security-framework 3.5.1", "serde", "serde_json", "sha2", + "subtle", "tempfile", "termimad", "testcontainers-modules", @@ -2332,10 +2646,11 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-tungstenite 0.26.2", - "tower", - "tower-http", + "tower 0.5.3", + "tower-http 0.6.8", "tracing", "tracing-subscriber", + "url", "urlencoding", "uuid", "wasmparser 0.220.1", @@ -2453,6 +2768,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128" version = "0.2.5" @@ -2471,6 +2792,16 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link 0.2.1", +] + [[package]] name = "libm" version = "0.2.16" @@ -2488,6 +2819,121 @@ dependencies = [ "redox_syscall 0.7.0", ] +[[package]] +name = "libsql" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe18646e4ef8db446bc3e3f5fb96131483203bc5f4998ff149f79a067530c01c" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "bincode", + "bitflags 2.10.0", + "bytes", + "fallible-iterator 0.3.0", + "futures", + "http 0.2.12", + "hyper 0.14.32", + "libsql-sqlite3-parser", + "libsql-sys", + "libsql_replication", + "parking_lot", + "serde", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tonic", + "tonic-web", + "tower 0.4.13", + "tower-http 0.4.4", + "tracing", + "uuid", + "zerocopy 0.7.35", +] + +[[package]] +name = "libsql-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2a50a585a1184a43621a9133b7702ba5cb7a87ca5e704056b19d8005de6faf" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "libsql-rusqlite" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae65c66088dcd309abbd5617ae046abac2a2ee0a7fdada5127353bd68e0a27ea" +dependencies = [ + "bitflags 2.10.0", + "fallible-iterator 0.2.0", + "fallible-streaming-iterator", + "hashlink", + "libsql-ffi", + "smallvec", +] + +[[package]] +name = "libsql-sqlite3-parser" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a90128c708356af8f7d767c9ac2946692c9112b4f74f07b99a01a60680e413" +dependencies = [ + "bitflags 2.10.0", + "cc", + "fallible-iterator 0.3.0", + "indexmap 2.13.0", + "log", + "memchr", + "phf 0.11.3", + "phf_codegen", + "phf_shared 0.11.3", + "uncased", +] + +[[package]] +name = "libsql-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c05b61c226781d6f5e26e3e7364617f19c0c1d5332035802e9229d6024cec05" +dependencies = [ + "bytes", + "libsql-ffi", + "libsql-rusqlite", + "once_cell", + "tracing", + "zerocopy 0.7.35", +] + +[[package]] +name = "libsql_replication" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf40c4c2c01462da758272976de0a23d19b4e9c714db08efecf262d896655b5" +dependencies = [ + "aes", + "async-stream", + "async-trait", + "bytes", + "cbc", + "libsql-rusqlite", + "libsql-sys", + "parking_lot", + "prost", + "serde", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tracing", + "uuid", + "zerocopy 0.7.35", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2551,6 +2997,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -2603,6 +3055,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimad" version = "0.14.0" @@ -2612,6 +3074,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.1.1" @@ -2624,6 +3092,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nanoid" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ffa00dec017b5b1a8b7cf5e2c008bfda1aa7e0697ac1508b491fdf2622fb4d8" +dependencies = [ + "rand 0.8.5", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.1.6", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -2658,6 +3152,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2806,18 +3310,71 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -2894,6 +3451,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2910,16 +3473,55 @@ dependencies = [ "postgres-types", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared", + "phf_shared 0.13.1", "serde", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", + "uncased", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -2929,6 +3531,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3050,7 +3672,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.37", ] [[package]] @@ -3063,6 +3685,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.114", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -3081,6 +3713,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "psm" version = "0.1.29" @@ -3133,9 +3788,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -3153,7 +3808,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -3172,7 +3827,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -3411,7 +4066,7 @@ dependencies = [ "bumpalo", "hashbrown 0.15.5", "log", - "rustc-hash", + "rustc-hash 2.1.1", "smallvec", ] @@ -3461,37 +4116,76 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "mime", + "mime_guess", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", - "tower", - "tower-http", + "tower 0.5.3", + "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", +] + +[[package]] +name = "rig-core" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f7a3f0c7c00eaced15a68ee16e1bd6bb709ff598d11b9aedac8b628217dc09" +dependencies = [ + "as-any", + "async-stream", + "base64 0.22.1", + "bytes", + "eventsource-stream", + "fastrand", + "futures", + "futures-timer", + "glob", + "http 1.4.0", + "mime", + "mime_guess", + "nanoid", + "ordered-float", + "pin-project-lite", + "reqwest", + "schemars 1.2.1", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-futures", + "url", ] [[package]] @@ -3570,6 +4264,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -3641,10 +4341,10 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.1", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.5.1", ] [[package]] @@ -3761,10 +4461,23 @@ checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", + "schemars_derive", "serde", "serde_json", ] +[[package]] +name = "schemars_derive" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.114", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -3806,6 +4519,19 @@ dependencies = [ "zbus", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.5.1" @@ -3813,7 +4539,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3869,6 +4595,17 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "serde_json" version = "1.0.149" @@ -4060,6 +4797,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -4162,6 +4909,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4182,6 +4935,27 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-interface" version = "0.27.3" @@ -4403,12 +5177,22 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "tracing", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.6.0" @@ -4420,6 +5204,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.16" @@ -4435,12 +5229,12 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", "rand 0.9.2", - "socket2", + "socket2 0.6.2", "tokio", "tokio-util", "whoami", @@ -4602,6 +5396,73 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-web" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3b0e1cedbf19fdfb78ef3d672cb9928e0a91a9cb4629cc0c916e8cff8aaaa1" +dependencies = [ + "base64 0.21.7", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "pin-project", + "tokio-stream", + "tonic", + "tower-http 0.4.4", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -4611,13 +5472,33 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "http-range-header", + "pin-project-lite", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-http" version = "0.6.8" @@ -4627,11 +5508,11 @@ dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "iri-string", "pin-project-lite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -4682,6 +5563,18 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "futures", + "futures-task", + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -4738,7 +5631,7 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -4755,7 +5648,7 @@ checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -4781,6 +5674,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -4903,6 +5811,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -5427,12 +6341,15 @@ dependencies = [ ] [[package]] -name = "webpki-roots" -version = "1.0.5" +name = "which" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ - "rustls-pki-types", + "either", + "home", + "once_cell", + "rustix 0.38.44", ] [[package]] @@ -5605,6 +6522,17 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.3.4" @@ -6060,13 +6988,34 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.35", +] + [[package]] name = "zerocopy" version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.8.37", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 91327a33..07428371 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,19 @@ [package] name = "ironclaw" -version = "0.1.0" +version = "0.1.3" edition = "2024" -rust-version = "1.85" +rust-version = "1.92" description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly" +authors = ["NEAR AI "] license = "MIT OR Apache-2.0" +homepage = "https://github.com/nearai/ironclaw" +repository = "https://github.com/nearai/ironclaw" + +[package.metadata.wix] +upgrade-guid = "D0156E61-BA37-451E-8AB9-1A2ECCCFA48F" +path-guid = "F90B6EA6-87F7-499B-BB19-CF55DE1EB339" +license = false +eula = false [dependencies] # Async runtime @@ -13,17 +22,20 @@ tokio-stream = { version = "0.1", features = ["sync"] } futures = "0.3" # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } # Serialization serde = { version = "1", features = ["derive"] } serde_json = "1" -# Database -deadpool-postgres = "0.14" -tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"] } -postgres-types = { version = "0.2", features = ["with-serde_json-1"] } -refinery = { version = "0.8", features = ["tokio-postgres"] } +# Database - PostgreSQL (default, feature-gated) +deadpool-postgres = { version = "0.14", optional = true } +tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true } +postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true } +refinery = { version = "0.8", features = ["tokio-postgres"], optional = true } + +# Database - libSQL/Turso (optional embedded database) +libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] } # Error handling thiserror = "2" @@ -39,7 +51,7 @@ dotenvy = "0.15" # Core types uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", features = ["serde"] } -rust_decimal = { version = "1", features = ["serde", "serde-with-str", "db-tokio-postgres", "maths"] } +rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] } rust_decimal_macros = "1" # Async traits @@ -58,17 +70,22 @@ axum = { version = "0.8", features = ["ws"] } tower = "0.5" tower-http = { version = "0.6", features = ["trace", "cors"] } +# Cron scheduling for routines +cron = "0.13" + # Safety/sanitization regex = "1" aho-corasick = "1" # Filesystem paths dirs = "6" +fs4 = "0.6" # Secrecy for sensitive values secrecy = { version = "0.10", features = ["serde"] } -# URL encoding for OAuth flow +# URL parsing and encoding +url = "2" urlencoding = "2" # Open URLs in browser @@ -76,7 +93,7 @@ open = "5" # Vector embeddings for semantic search # The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres) -pgvector = { version = "0.4", features = ["postgres"] } +pgvector = { version = "0.4", features = ["postgres"], optional = true } # WASM sandbox for untrusted tool execution wasmtime = { version = "28", features = ["component-model"] } @@ -89,6 +106,10 @@ hkdf = "0.12" sha2 = "0.10" blake3 = "1" rand = "0.8" +subtle = "2" # Constant-time comparisons for token validation + +# Multi-provider LLM support +rig-core = "0.30" # Docker sandbox bollard = "0.18" @@ -99,6 +120,7 @@ hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] http-body-util = "0.1" bytes = "1" base64 = "0.22.1" +mime_guess = "2.0.5" # Headless browser automation via Chrome DevTools Protocol chromiumoxide = { version = "0.8", default-features = false, features = ["tokio-runtime"] } @@ -120,5 +142,59 @@ pretty_assertions = "1" tempfile = "3" [features] -default = [] +default = ["postgres"] +postgres = [ + "dep:deadpool-postgres", + "dep:tokio-postgres", + "dep:postgres-types", + "dep:refinery", + "dep:pgvector", + "rust_decimal/db-tokio-postgres", +] +libsql = ["dep:libsql"] integration = [] + +[[example]] +name = "test_heartbeat" +required-features = ["postgres"] + +# The profile that 'cargo dist' will build with +[profile.dist] +inherits = "release" +lto = "thin" + +# Config for 'dist' +[workspace.metadata.dist] +# The preferred dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.30.3" +# CI backends to support +ci = "github" +# The installers to generate for each app +installers = ["shell", "powershell", "npm", "msi"] +# Publish jobs to run in CI +publish-jobs = [] +# Target platforms to build apps for (Rust target-triple syntax) +targets = [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu", + "x86_64-pc-windows-msvc", +] +# The archive format to use for windows builds (defaults .zip) +windows-archive = ".tar.gz" +# The archive format to use for non-windows builds (defaults .tar.xz) +unix-archive = ".tar.gz" +# Which actions to run on pull requests +pr-run-mode = "upload" +# Path that installers should place binaries in +install-path = "CARGO_HOME" +# Whether to install an updater program +install-updater = true + +[workspace.metadata.dist.github-custom-runners] +aarch64-unknown-linux-gnu = "ubuntu-24.04-arm" +x86_64-unknown-linux-gnu = "ubuntu-22.04" +x86_64-pc-windows-msvc = "windows-2022" +x86_64-apple-darwin = "macos-15-intel" +aarch64-apple-darwin = "macos-14" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..34d4d484 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +# Multi-stage Dockerfile for the IronClaw agent (cloud deployment). +# +# Build: +# docker build --platform linux/amd64 -t ironclaw:latest . +# +# Run: +# docker run --env-file .env -p 3000:3000 ironclaw:latest + +# Stage 1: Build +FROM rust:1.92-slim-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev cmake gcc g++ \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy manifests first for layer caching +COPY Cargo.toml Cargo.lock ./ + +# Copy source and build artifacts +COPY src/ src/ +COPY migrations/ migrations/ +COPY wit/ wit/ + +RUN cargo build --release --bin ironclaw + +# Stage 2: Runtime +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libssl3 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/target/release/ironclaw /usr/local/bin/ironclaw +COPY --from=builder /app/migrations /app/migrations + +# Non-root user +RUN useradd -m -u 1000 -s /bin/bash ironclaw +USER ironclaw + +EXPOSE 3000 + +ENV RUST_LOG=ironclaw=info + +ENTRYPOINT ["ironclaw"] diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 00000000..6c58a5e5 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,63 @@ +# Multi-stage Dockerfile for the IronClaw worker container. +# +# This image runs the ironclaw binary in worker mode inside Docker containers. +# The orchestrator creates instances of this image for sandboxed job execution. +# +# Build: +# docker build -f Dockerfile.worker -t ironclaw-worker . +# +# The image includes common development tools so workers can build software, +# run tests, and execute shell commands. + +FROM rust:1.92-bookworm AS builder + +WORKDIR /build +COPY . . + +# Build only the ironclaw binary (release mode) +RUN cargo build --release --bin ironclaw + +# --- + +FROM debian:bookworm-slim + +# Install common development tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + build-essential \ + pkg-config \ + libssl-dev \ + nodejs \ + npm \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust toolchain for the sandbox user +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.92.0 \ + && chmod -R a+r /usr/local/rustup /usr/local/cargo + +# Install Claude Code CLI (for claude-bridge mode) +RUN npm install -g @anthropic-ai/claude-code@latest + +# Copy the binary +COPY --from=builder /build/target/release/ironclaw /usr/local/bin/ironclaw + +# Create non-root user (UID 1000 matches the orchestrator's container config) +RUN useradd -m -u 1000 -s /bin/bash sandbox \ + && mkdir -p /workspace \ + && chown sandbox:sandbox /workspace \ + && mkdir -p /home/sandbox/.claude \ + && chown sandbox:sandbox /home/sandbox/.claude + +USER sandbox +WORKDIR /workspace + +# The orchestrator passes the full command via Docker cmd. +ENTRYPOINT ["ironclaw"] diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index 6f791052..cda8dfd1 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -16,8 +16,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Hub-and-spoke architecture | ✅ | 🚧 | IronClaw has channels but no central gateway | -| WebSocket control plane | ✅ | ❌ | Gateway with ws://127.0.0.1:18789 | +| Hub-and-spoke architecture | ✅ | ✅ | Web gateway as central hub | +| WebSocket control plane | ✅ | ✅ | Gateway with WebSocket + SSE | | Single-user system | ✅ | ✅ | | | Multi-agent routing | ✅ | ❌ | Workspace isolation per-agent | | Session-based messaging | ✅ | ✅ | Per-sender sessions | @@ -31,19 +31,19 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Gateway control plane | ✅ | ❌ | Central WebSocket server | -| HTTP endpoints for Control UI | ✅ | ❌ | Web dashboard | -| Channel connection lifecycle | ✅ | 🚧 | ChannelManager handles streams | +| Gateway control plane | ✅ | ✅ | Web gateway with 40+ API endpoints | +| HTTP endpoints for Control UI | ✅ | ✅ | Web dashboard with chat, memory, jobs, logs, extensions | +| Channel connection lifecycle | ✅ | ✅ | ChannelManager + WebSocket tracker | | Session management/routing | ✅ | ✅ | SessionManager exists | | Configuration hot-reload | ✅ | ❌ | | | Network modes (loopback/LAN/remote) | ✅ | 🚧 | HTTP only | -| OpenAI-compatible HTTP API | ✅ | ❌ | /v1/chat/completions | +| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions | | Canvas hosting | ✅ | ❌ | Agent-driven UI | | Gateway lock (PID-based) | ✅ | ❌ | | | launchd/systemd integration | ✅ | ❌ | | | Bonjour/mDNS discovery | ✅ | ❌ | | | Tailscale integration | ✅ | ❌ | | -| Health check endpoints | ✅ | ❌ | | +| Health check endpoints | ✅ | ✅ | /api/health + /api/gateway/status | | `doctor` diagnostics | ✅ | ❌ | | ### Owner: _Unassigned_ @@ -59,14 +59,14 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | REPL (simple) | ✅ | ✅ | - | For testing | | WASM channels | ❌ | ✅ | - | IronClaw innovation | | WhatsApp | ✅ | ❌ | P1 | Baileys (Web) | -| Telegram | ✅ | ❌ | P1 | grammY (Bot API) | +| Telegram | ✅ | ✅ | - | WASM channel(MTProto), DM pairing, caption, /start, bot_username | | Discord | ✅ | ❌ | P2 | discord.js | | Signal | ✅ | ❌ | P2 | signal-cli | -| Slack | ✅ | 🚧 | P1 | Stub exists, needs implementation | +| Slack | ✅ | ✅ | - | WASM tool | | iMessage | ✅ | ❌ | P3 | BlueBubbles recommended | | Feishu/Lark | ✅ | ❌ | P3 | | | LINE | ✅ | ❌ | P3 | | -| WebChat | ✅ | ❌ | P2 | Browser-based chat | +| WebChat | ✅ | ✅ | - | Web gateway chat | | Matrix | ✅ | ❌ | P3 | E2EE support | | Mattermost | ✅ | ❌ | P3 | | | Google Chat | ✅ | ❌ | P3 | | @@ -79,13 +79,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| DM pairing codes | ✅ | ❌ | Verification for unknown senders | -| Allowlist/blocklist | ✅ | ❌ | Per-channel access control | +| DM pairing codes | ✅ | ✅ | `ironclaw pairing list/approve`, host APIs | +| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store | | Self-message bypass | ✅ | ❌ | Own messages skip pairing | -| Mention-based activation | ✅ | ❌ | Configurable patterns | +| Mention-based activation | ✅ | ✅ | bot_username + respond_to_all_group_messages | | Per-group tool policies | ✅ | ❌ | Allow/deny specific tools | | Thread isolation | ✅ | ✅ | Separate sessions per thread | -| Per-channel media limits | ✅ | ❌ | | +| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits | | Typing indicators | ✅ | 🚧 | TUI shows status | ### Owner: _Unassigned_ @@ -99,17 +99,17 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | `run` (agent) | ✅ | ✅ | - | Default command | | `tool install/list/remove` | ✅ | ✅ | - | WASM tools | | `gateway start/stop` | ✅ | ❌ | P2 | | -| `onboard` (wizard) | ✅ | ❌ | P2 | Interactive setup | +| `onboard` (wizard) | ✅ | ✅ | - | Interactive setup | | `tui` | ✅ | ✅ | - | Ratatui TUI | -| `config` | ✅ | ❌ | P2 | Read/write config | +| `config` | ✅ | ✅ | - | Read/write config | | `channels` | ✅ | ❌ | P2 | Channel management | | `models` | ✅ | 🚧 | - | Model selector in TUI | -| `status` | ✅ | ❌ | P2 | System status | +| `status` | ✅ | ✅ | - | System status | | `agents` | ✅ | ❌ | P3 | Multi-agent management | | `sessions` | ✅ | ❌ | P3 | Session listing | -| `memory` | ✅ | ❌ | P2 | Memory search CLI | +| `memory` | ✅ | ✅ | - | Memory search CLI | | `skills` | ✅ | ❌ | P3 | Agent skills | -| `pairing` | ✅ | ❌ | P3 | Node pairing | +| `pairing` | ✅ | ✅ | - | list/approve for channel DM pairing | | `nodes` | ✅ | ❌ | P3 | Device management | | `plugins` | ✅ | ❌ | P3 | Plugin management | | `hooks` | ✅ | ❌ | P2 | Lifecycle hooks | @@ -132,8 +132,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| | Pi agent runtime | ✅ | ➖ | IronClaw uses custom runtime | -| RPC-based execution | ✅ | 🚧 | Worker isolation | -| Multi-provider failover | ✅ | ❌ | Provider fallback chains | +| RPC-based execution | ✅ | ✅ | Orchestrator/worker pattern | +| Multi-provider failover | ✅ | ✅ | `FailoverProvider` tries providers sequentially on retryable errors | | Per-sender sessions | ✅ | ✅ | | | Global sessions | ✅ | ❌ | Optional shared context | | Session pruning | ✅ | ❌ | Auto cleanup old sessions | @@ -173,7 +173,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| | Auto-discovery | ✅ | ❌ | | -| Failover chains | ✅ | ❌ | Provider fallback | +| Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` | | Cooldown management | ✅ | ❌ | Skip failed providers | | Per-session model override | ✅ | ✅ | Model selector in TUI | | Model selection UI | ✅ | ✅ | TUI keyboard shortcut | @@ -303,13 +303,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| -| Control UI Dashboard | ✅ | ❌ | P2 | Web status/config | -| Channel status view | ✅ | ❌ | P2 | | +| Control UI Dashboard | ✅ | ✅ | - | Web gateway with chat, memory, jobs, logs, extensions | +| Channel status view | ✅ | 🚧 | P2 | Gateway status widget, full channel view pending | | Agent management | ✅ | ❌ | P3 | | | Model selection | ✅ | ✅ | - | TUI only | | Config editing | ✅ | ❌ | P3 | | -| Debug/logs viewer | ✅ | ❌ | P3 | | -| WebChat interface | ✅ | ❌ | P2 | Browser chat | +| Debug/logs viewer | ✅ | ✅ | - | Real-time log streaming with level/target filters | +| WebChat interface | ✅ | ✅ | - | Web gateway chat with SSE/WebSocket | | Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI | ### Owner: _Unassigned_ @@ -320,13 +320,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| -| Cron jobs | ✅ | ❌ | P2 | Schedule-based tasks | -| Timezone support | ✅ | ❌ | P2 | | -| One-shot/recurring jobs | ✅ | ❌ | P2 | | +| Cron jobs | ✅ | ✅ | - | Routines with cron trigger | +| Timezone support | ✅ | ✅ | - | Via cron expressions | +| One-shot/recurring jobs | ✅ | ✅ | - | Manual + cron triggers | | `beforeInbound` hook | ✅ | ❌ | P2 | | | `beforeOutbound` hook | ✅ | ❌ | P2 | | | `beforeToolCall` hook | ✅ | ❌ | P2 | | -| `onMessage` hook | ✅ | ❌ | P2 | | +| `onMessage` hook | ✅ | ✅ | - | Routines with event trigger | | `onSessionStart` hook | ✅ | ❌ | P2 | | | `onSessionEnd` hook | ✅ | ❌ | P2 | | | `transcribeAudio` hook | ✅ | ❌ | P3 | | @@ -346,18 +346,18 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| -| Gateway token auth | ✅ | 🚧 | HTTP webhook secret | +| Gateway token auth | ✅ | ✅ | Bearer token auth on web gateway | | Device pairing | ✅ | ❌ | | | Tailscale identity | ✅ | ❌ | | | OAuth flows | ✅ | 🚧 | NEAR AI OAuth | -| DM pairing verification | ✅ | ❌ | | -| Allowlist/blocklist | ✅ | ❌ | | +| DM pairing verification | ✅ | ✅ | ironclaw pairing approve, host APIs | +| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store | | Per-group tool policies | ✅ | ❌ | | | Exec approvals | ✅ | ✅ | TUI overlay | | TLS 1.3 minimum | ✅ | ✅ | reqwest rustls | | SSRF protection | ✅ | ✅ | WASM allowlist | | Loopback-first | ✅ | 🚧 | HTTP binds 0.0.0.0 | -| Docker sandbox | ✅ | ❌ | Uses WASM sandbox | +| Docker sandbox | ✅ | ✅ | Orchestrator/worker containers | | WASM sandbox | ❌ | ✅ | IronClaw innovation | | Tool policies | ✅ | ✅ | | | Elevated mode | ✅ | ❌ | | @@ -397,6 +397,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O ### P0 - Core (Already Done) - ✅ TUI channel with approval overlays - ✅ HTTP webhook channel +- ✅ DM pairing (ironclaw pairing list/approve, host APIs) - ✅ WASM tool sandbox - ✅ Workspace/memory with hybrid search - ✅ Prompt injection defense @@ -404,23 +405,32 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - ✅ Session management - ✅ Context compaction - ✅ Model selection +- ✅ Gateway control plane + WebSocket +- ✅ Web Control UI (chat, memory, jobs, logs, extensions, routines) +- ✅ WebChat channel (web gateway) +- ✅ Slack channel (WASM tool) +- ✅ Telegram channel (WASM tool, MTProto) +- ✅ Docker sandbox (orchestrator/worker) +- ✅ Cron job scheduling (routines) +- ✅ CLI subcommands (onboard, config, status, memory) +- ✅ Gateway token auth ### P1 - High Priority - ❌ Slack channel (real implementation) -- ❌ Telegram channel +- ✅ Telegram channel (WASM, DM pairing, caption, /start) - ❌ WhatsApp channel -- ❌ Multi-provider failover -- ❌ Gateway control plane + WebSocket +- ✅ Multi-provider failover (`FailoverProvider` with retryable error classification) - ❌ Hooks system (beforeInbound, beforeToolCall, etc.) ### P2 - Medium Priority - ❌ Cron job scheduling - ❌ Web Control UI - ❌ WebChat channel -- ❌ Media handling (images, PDFs) +- 🚧 Media handling (caption support; no image/PDF processing) - ❌ CLI subcommands (config, status, memory, doctor) - ❌ Ollama/local model support - ❌ Configuration hot-reload +- ❌ Webhook trigger endpoint in web gateway ### P3 - Lower Priority - ❌ Discord channel diff --git a/README.md b/README.md index ce78526a..d8fc7a78 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,10 @@ IronClaw is the AI assistant you can actually trust with your personal and profe ### Always Available -- **Multi-channel** - REPL, HTTP webhooks, and extensible WASM channels (Telegram, Slack, and more) +- **Multi-channel** - REPL, HTTP webhooks, WASM channels (Telegram, Slack), and web gateway +- **Docker Sandbox** - Isolated container execution with per-job tokens and orchestrator/worker pattern +- **Web Gateway** - Browser UI with real-time SSE/WebSocket streaming +- **Routines** - Cron schedules, event triggers, webhook handlers for background automation - **Heartbeat System** - Proactive background execution for monitoring and maintenance tasks - **Parallel Jobs** - Handle multiple requests concurrently with isolated contexts - **Self-repair** - Automatic detection and recovery of stuck operations @@ -65,10 +68,41 @@ IronClaw is the AI assistant you can actually trust with your personal and profe ### Prerequisites - Rust 1.85+ -- PostgreSQL 15+ with pgvector extension +- PostgreSQL 15+ with [pgvector](https://github.com/pgvector/pgvector) extension - NEAR AI account (authentication handled via setup wizard) -### Build +## Download or Build + +Visit [Releases page](https://github.com/nearai/ironclaw/releases/) to see the latest updates. + +
+ Install via Windows Installer (Windows) + +Download the [Windows Installer](https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-x86_64-pc-windows-msvc.msi) and run it. + +
+ +
+ Install via powershell script (Windows) + +```sh +irm https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.ps1 | iex +``` + +
+ +
+ Install via shell script (macOS, Linux, Windows/WSL) + +```sh +curl --proto '=https' --tlsv1.2 -LsSf https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.sh | sh +``` +
+ +
+ Compile the source code (Cargo on Windows, Linux, macOS) + +Install it with `cargo`, just make sure you have [Rust](https://rustup.rs) installed on your computer. ```bash # Clone the repository @@ -82,6 +116,10 @@ cargo build --release cargo test ``` +For **full release** (after modifying channel sources), run `./scripts/build-all.sh` to rebuild channels first. + +
+ ### Database Setup ```bash @@ -97,7 +135,7 @@ psql ironclaw -c "CREATE EXTENSION IF NOT EXISTS vector;" Run the setup wizard to configure IronClaw: ```bash -ironclaw setup +ironclaw onboard ``` The wizard handles database connection, NEAR AI authentication (via browser OAuth), @@ -143,37 +181,42 @@ External content passes through multiple security layers: ## Architecture ``` -┌─────────────────────────────────────────────────────────────────┐ -│ Channels │ -│ ┌──────┐ ┌──────┐ ┌──────────────┐ │ -│ │ REPL │ │ HTTP │ │ WASM Channels│ │ -│ └──┬───┘ └──┬───┘ └──────┬───────┘ │ -│ └─────────┴─────────────┘ │ -│ │ │ -│ ┌────▼────┐ │ -│ │ Router │ Intent classification │ -│ └────┬────┘ │ -│ │ │ -│ ┌──────────▼──────────┐ │ -│ │ Scheduler │ Parallel job management │ -│ └──────────┬──────────┘ │ -│ │ │ -│ ┌───────────────┼───────────────┐ │ -│ ▼ ▼ ▼ │ -│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │ -│ │ Worker │ │ Worker │ │ Worker │ LLM reasoning │ -│ └────┬────┘ └────┬────┘ └────┬────┘ │ -│ └───────────────┼───────────────┘ │ -│ │ │ -│ ┌──────────▼──────────┐ │ -│ │ Tool Registry │ │ -│ │ ┌───────────────┐ │ │ -│ │ │ Built-in │ │ │ -│ │ │ MCP │ │ │ -│ │ │ WASM Sandbox │ │ │ -│ │ └───────────────┘ │ │ -│ └─────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ +┌────────────────────────────────────────────────────────────────┐ +│ Channels │ +│ ┌──────┐ ┌──────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ REPL │ │ HTTP │ │WASM Channels│ │ Web Gateway │ │ +│ └──┬───┘ └──┬───┘ └──────┬──────┘ │ (SSE + WS) │ │ +│ │ │ │ └──────┬──────┘ │ +│ └─────────┴──────────────┴────────────────┘ │ +│ │ │ +│ ┌─────────▼─────────┐ │ +│ │ Agent Loop │ Intent routing │ +│ └────┬──────────┬───┘ │ +│ │ │ │ +│ ┌──────────▼────┐ ┌──▼───────────────┐ │ +│ │ Scheduler │ │ Routines Engine │ │ +│ │(parallel jobs)│ │(cron, event, wh) │ │ +│ └──────┬────────┘ └────────┬─────────┘ │ +│ │ │ │ +│ ┌─────────────┼────────────────────┘ │ +│ │ │ │ +│ ┌───▼─────┐ ┌────▼────────────────┐ │ +│ │ Local │ │ Orchestrator │ │ +│ │Workers │ │ ┌───────────────┐ │ │ +│ │(in-proc)│ │ │ Docker Sandbox│ │ │ +│ └───┬─────┘ │ │ Containers │ │ │ +│ │ │ │ ┌───────────┐ │ │ │ +│ │ │ │ │Worker / CC│ │ │ │ +│ │ │ │ └───────────┘ │ │ │ +│ │ │ └───────────────┘ │ │ +│ │ └─────────┬───────────┘ │ +│ └──────────────────┤ │ +│ │ │ +│ ┌───────────▼──────────┐ │ +│ │ Tool Registry │ │ +│ │ Built-in, MCP, WASM │ │ +│ └──────────────────────┘ │ +└────────────────────────────────────────────────────────────────┘ ``` ### Core Components @@ -184,6 +227,9 @@ External content passes through multiple security layers: | **Router** | Classifies user intent (command, query, task) | | **Scheduler** | Manages parallel job execution with priorities | | **Worker** | Executes jobs with LLM reasoning and tool calls | +| **Orchestrator** | Container lifecycle, LLM proxying, per-job auth | +| **Web Gateway** | Browser UI with chat, memory, jobs, logs, extensions, routines | +| **Routines Engine** | Scheduled (cron) and reactive (event, webhook) background tasks | | **Workspace** | Persistent memory with hybrid search | | **Safety Layer** | Prompt injection defense and content sanitization | @@ -191,7 +237,7 @@ External content passes through multiple security layers: ```bash # First-time setup (configures database, auth, etc.) -ironclaw setup +ironclaw onboard # Start interactive REPL cargo run @@ -210,12 +256,16 @@ cargo fmt cargo clippy --all --benches --tests --examples --all-features # Run tests +createdb ironclaw_test cargo test # Run specific test cargo test test_name ``` +- **Telegram channel**: See [docs/TELEGRAM_SETUP.md](docs/TELEGRAM_SETUP.md) for setup and DM pairing. +- **Changing channel sources**: Run `./channels-src/telegram/build.sh` before `cargo build` so the updated WASM is bundled. + ## OpenClaw Heritage IronClaw is a Rust reimplementation inspired by [OpenClaw](https://github.com/openclaw/openclaw). See [FEATURE_PARITY.md](FEATURE_PARITY.md) for the complete tracking matrix. diff --git a/build.rs b/build.rs new file mode 100644 index 00000000..8f695ee0 --- /dev/null +++ b/build.rs @@ -0,0 +1,106 @@ +//! Build script: compile Telegram channel WASM from source. +//! +//! Do not commit compiled WASM binaries — they are a supply chain risk. +//! This script builds telegram.wasm from channels-src/telegram before the main crate compiles. +//! +//! Reproducible build: +//! cargo build --release +//! (build.rs invokes the channel build automatically) +//! +//! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let root = PathBuf::from(&manifest_dir); + let channel_dir = root.join("channels-src/telegram"); + let wasm_out = channel_dir.join("telegram.wasm"); + + // Rerun when channel source or build script changes + println!("cargo:rerun-if-changed=channels-src/telegram/src"); + println!("cargo:rerun-if-changed=channels-src/telegram/Cargo.toml"); + println!("cargo:rerun-if-changed=wit/channel.wit"); + + if !channel_dir.is_dir() { + return; + } + + // Build WASM module + let status = match Command::new("cargo") + .args([ + "build", + "--release", + "--target", + "wasm32-wasip2", + "--manifest-path", + channel_dir.join("Cargo.toml").to_str().unwrap(), + ]) + .current_dir(&root) + .status() + { + Ok(s) => s, + Err(_) => { + eprintln!( + "cargo:warning=Telegram channel build failed. Run: ./channels-src/telegram/build.sh" + ); + return; + } + }; + + if !status.success() { + eprintln!( + "cargo:warning=Telegram channel build failed. Run: ./channels-src/telegram/build.sh" + ); + return; + } + + let raw_wasm = channel_dir.join("target/wasm32-wasip2/release/telegram_channel.wasm"); + if !raw_wasm.exists() { + eprintln!( + "cargo:warning=Telegram WASM output not found at {:?}", + raw_wasm + ); + return; + } + + // Convert to component and strip (wasm-tools) + let component_ok = Command::new("wasm-tools") + .args([ + "component", + "new", + raw_wasm.to_str().unwrap(), + "-o", + wasm_out.to_str().unwrap(), + ]) + .current_dir(&root) + .status() + .map(|s| s.success()) + .unwrap_or(false); + + if !component_ok { + // Fallback: copy raw module if wasm-tools unavailable + if std::fs::copy(&raw_wasm, &wasm_out).is_err() { + eprintln!("cargo:warning=wasm-tools not found. Run: cargo install wasm-tools"); + } + } else { + // Strip debug info (use temp file to avoid clobbering) + let stripped = wasm_out.with_extension("wasm.stripped"); + let strip_ok = Command::new("wasm-tools") + .args([ + "strip", + wasm_out.to_str().unwrap(), + "-o", + stripped.to_str().unwrap(), + ]) + .current_dir(&root) + .status() + .map(|s| s.success()) + .unwrap_or(false); + if strip_ok { + let _ = std::fs::rename(&stripped, &wasm_out); + } + } +} diff --git a/channels-src/slack/build.sh b/channels-src/slack/build.sh index 5b568c8d..5ab678f9 100755 --- a/channels-src/slack/build.sh +++ b/channels-src/slack/build.sh @@ -30,7 +30,7 @@ if [ -f "$WASM_PATH" ]; then wasm-tools strip slack.wasm -o slack.wasm echo "Built: slack.wasm ($(du -h slack.wasm | cut -f1))" - echo "Copy slack.wasm and slack.capabilities.json to ~/.near-agent/channels/" + echo "Copy slack.wasm and slack.capabilities.json to ~/.ironclaw/channels/" else echo "Error: WASM output not found at $WASM_PATH" exit 1 diff --git a/channels-src/slack/src/lib.rs b/channels-src/slack/src/lib.rs index c54af12f..5cf7b10d 100644 --- a/channels-src/slack/src/lib.rs +++ b/channels-src/slack/src/lib.rs @@ -108,7 +108,10 @@ struct SlackPostMessageResponse { #[derive(Debug, Deserialize)] struct SlackConfig { /// Name of secret containing signing secret (for verification by host). + /// Parsed from config for forward compatibility; not yet used in WASM + /// (host handles signature verification). #[serde(default = "default_signing_secret_name")] + #[allow(dead_code)] signing_secret_name: String, } @@ -175,11 +178,7 @@ impl Guest for SlackChannel { // Actual event callback "event_callback" => { if let Some(event) = event_wrapper.event { - handle_slack_event( - event, - event_wrapper.team_id, - event_wrapper.event_id, - ); + handle_slack_event(event, event_wrapper.team_id, event_wrapper.event_id); } // Always respond 200 quickly to Slack (they have a 3s timeout) json_response(200, serde_json::json!({"ok": true})) @@ -230,6 +229,7 @@ impl Guest for SlackChannel { "https://slack.com/api/chat.postMessage", &headers.to_string(), Some(&payload_bytes), + None, ); match result { @@ -243,14 +243,15 @@ impl Guest for SlackChannel { // Parse Slack response let slack_response: SlackPostMessageResponse = - serde_json::from_slice(&http_response.body).map_err(|e| { - format!("Failed to parse Slack response: {}", e) - })?; + serde_json::from_slice(&http_response.body) + .map_err(|e| format!("Failed to parse Slack response: {}", e))?; if !slack_response.ok { return Err(format!( "Slack API error: {}", - slack_response.error.unwrap_or_else(|| "unknown".to_string()) + slack_response + .error + .unwrap_or_else(|| "unknown".to_string()) )); } @@ -277,17 +278,16 @@ impl Guest for SlackChannel { } /// Handle a Slack event and emit message if applicable. -fn handle_slack_event( - event: SlackEvent, - team_id: Option, - _event_id: Option, -) { +fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Option) { match event.event_type.as_str() { // Direct mention of the bot "app_mention" => { - if let (Some(user), Some(channel), Some(text), Some(ts)) = - (event.user, event.channel.clone(), event.text, event.ts.clone()) - { + if let (Some(user), Some(channel), Some(text), Some(ts)) = ( + event.user, + event.channel.clone(), + event.text, + event.ts.clone(), + ) { emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); } } @@ -299,9 +299,12 @@ fn handle_slack_event( return; } - if let (Some(user), Some(channel), Some(text), Some(ts)) = - (event.user, event.channel.clone(), event.text, event.ts.clone()) - { + if let (Some(user), Some(channel), Some(text), Some(ts)) = ( + event.user, + event.channel.clone(), + event.text, + event.ts.clone(), + ) { // Only process DMs (channel IDs starting with D) if channel.starts_with('D') { emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); @@ -335,8 +338,7 @@ fn emit_message( team_id, }; - let metadata_json = - serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string()); + let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string()); // Strip @ mentions of the bot from the text for cleaner messages let cleaned_text = strip_bot_mention(&text); diff --git a/channels-src/telegram/build.sh b/channels-src/telegram/build.sh index 531ab762..3e4d7808 100755 --- a/channels-src/telegram/build.sh +++ b/channels-src/telegram/build.sh @@ -32,8 +32,8 @@ if [ -f "$WASM_PATH" ]; then echo "Built: telegram.wasm ($(du -h telegram.wasm | cut -f1))" echo "" echo "To install:" - echo " mkdir -p ~/.near-agent/channels" - echo " cp telegram.wasm telegram.capabilities.json ~/.near-agent/channels/" + echo " mkdir -p ~/.ironclaw/channels" + echo " cp telegram.wasm telegram.capabilities.json ~/.ironclaw/channels/" echo "" echo "Then add your bot token to secrets:" echo " # Set TELEGRAM_BOT_TOKEN in your environment or secrets store" diff --git a/channels-src/telegram/src/lib.rs b/channels-src/telegram/src/lib.rs index 984385bc..08e82804 100644 --- a/channels-src/telegram/src/lib.rs +++ b/channels-src/telegram/src/lib.rs @@ -72,6 +72,10 @@ struct TelegramMessage { /// Message text. text: Option, + /// Caption for media (photo, video, document, etc.). + #[serde(default)] + caption: Option, + /// Original message if this is a reply. reply_to_message: Option>, @@ -160,6 +164,21 @@ const POLLING_STATE_PATH: &str = "state/last_update_id"; /// Workspace path for persisting owner_id across WASM callbacks. const OWNER_ID_PATH: &str = "state/owner_id"; +/// Workspace path for persisting dm_policy across WASM callbacks. +const DM_POLICY_PATH: &str = "state/dm_policy"; + +/// Workspace path for persisting allow_from (JSON array) across WASM callbacks. +const ALLOW_FROM_PATH: &str = "state/allow_from"; + +/// Channel name for pairing store (used by pairing host APIs). +const CHANNEL_NAME: &str = "telegram"; + +/// Workspace path for persisting bot_username for mention detection in groups. +const BOT_USERNAME_PATH: &str = "state/bot_username"; + +/// Workspace path for persisting respond_to_all_group_messages flag. +const RESPOND_TO_ALL_GROUP_PATH: &str = "state/respond_to_all_group_messages"; + // ============================================================================ // Channel Metadata // ============================================================================ @@ -196,6 +215,14 @@ struct TelegramConfig { #[serde(default)] owner_id: Option, + /// DM policy: "pairing" (default), "allowlist", or "open". + #[serde(default)] + dm_policy: Option, + + /// Allowed sender IDs/usernames from config (merged with pairing-approved store). + #[serde(default)] + allow_from: Option>, + /// Whether to respond to all group messages (not just mentions). #[serde(default)] respond_to_all_group_messages: bool, @@ -257,6 +284,28 @@ impl Guest for TelegramChannel { ); } + // Persist dm_policy and allow_from for DM pairing in handle_message + let dm_policy = config + .dm_policy + .as_deref() + .unwrap_or("pairing") + .to_string(); + let _ = channel_host::workspace_write(DM_POLICY_PATH, &dm_policy); + + let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default()) + .unwrap_or_else(|_| "[]".to_string()); + let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json); + + // Persist bot_username and respond_to_all_group_messages for group handling + let _ = channel_host::workspace_write( + BOT_USERNAME_PATH, + &config.bot_username.unwrap_or_default(), + ); + let _ = channel_host::workspace_write( + RESPOND_TO_ALL_GROUP_PATH, + &config.respond_to_all_group_messages.to_string(), + ); + // Mode is determined by whether the host injected a tunnel_url // If tunnel is configured, use webhooks. Otherwise, use polling. let webhook_mode = config.tunnel_url.is_some(); @@ -388,7 +437,9 @@ impl Guest for TelegramChannel { let headers = serde_json::json!({}); - let result = channel_host::http_request("GET", &url, &headers.to_string(), None); + // 35s HTTP timeout outlives Telegram's 30s server-side long-poll + let result = + channel_host::http_request("GET", &url, &headers.to_string(), None, Some(35_000)); match result { Ok(response) => { @@ -461,72 +512,52 @@ impl Guest for TelegramChannel { } fn on_respond(response: AgentResponse) -> Result<(), String> { - // Parse metadata to get chat info let metadata: TelegramMessageMetadata = serde_json::from_str(&response.metadata_json) .map_err(|e| format!("Failed to parse metadata: {}", e))?; - // Build sendMessage payload - let mut payload = serde_json::json!({ - "chat_id": metadata.chat_id, - "text": response.content, - "parse_mode": "Markdown", - }); - - // Reply to the original message for context - payload["reply_to_message_id"] = serde_json::Value::Number(metadata.message_id.into()); - - let payload_bytes = serde_json::to_vec(&payload) - .map_err(|e| format!("Failed to serialize payload: {}", e))?; - - // Make HTTP request to Telegram API - // The bot token is injected into the URL by the host - let headers = serde_json::json!({ - "Content-Type": "application/json" - }); - - let result = channel_host::http_request( - "POST", - "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage", - &headers.to_string(), - Some(&payload_bytes), + // Try sending with Markdown first; fall back to plain text if Telegram + // can't parse the entities (e.g. model leaked with underscores). + let result = send_message( + metadata.chat_id, + &response.content, + metadata.message_id, + Some("Markdown"), ); match result { - Ok(http_response) => { - if http_response.status != 200 { - let body_str = String::from_utf8_lossy(&http_response.body); - return Err(format!( - "Telegram API returned status {}: {}", - http_response.status, body_str - )); - } - - // Parse Telegram response - let api_response: TelegramApiResponse = - serde_json::from_slice(&http_response.body) - .map_err(|e| format!("Failed to parse Telegram response: {}", e))?; - - if !api_response.ok { - return Err(format!( - "Telegram API error: {}", - api_response - .description - .unwrap_or_else(|| "unknown".to_string()) - )); - } - + Ok(msg_id) => { channel_host::log( channel_host::LogLevel::Debug, &format!( "Sent message to chat {}: message_id={}", - metadata.chat_id, - api_response.result.map(|r| r.message_id).unwrap_or(0) + metadata.chat_id, msg_id ), ); - Ok(()) } - Err(e) => Err(format!("HTTP request failed: {}", e)), + Err(SendError::ParseEntities(detail)) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!("Markdown parse failed ({}), retrying as plain text", detail), + ); + let msg_id = send_message( + metadata.chat_id, + &response.content, + metadata.message_id, + None, + ) + .map_err(|e| format!("Plain-text retry also failed: {}", e))?; + + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Sent plain-text message to chat {}: message_id={}", + metadata.chat_id, msg_id + ), + ); + Ok(()) + } + Err(e) => Err(e.to_string()), } } @@ -568,6 +599,7 @@ impl Guest for TelegramChannel { "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction", &headers.to_string(), Some(&payload_bytes), + None, ); if let Err(e) = result { @@ -586,6 +618,101 @@ impl Guest for TelegramChannel { } } +// ============================================================================ +// Send Message Helper +// ============================================================================ + +/// Errors from send_message, split so callers can match on parse-entity failures. +enum SendError { + /// Telegram returned 400 with "can't parse entities" (Markdown issue). + ParseEntities(String), + /// Any other failure. + Other(String), +} + +impl std::fmt::Display for SendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SendError::ParseEntities(detail) => write!(f, "parse entities error: {}", detail), + SendError::Other(msg) => write!(f, "{}", msg), + } + } +} + +/// Send a message via the Telegram Bot API. +/// +/// Returns the sent message_id on success. When `parse_mode` is set and +/// Telegram returns a 400 "can't parse entities" error, returns +/// `SendError::ParseEntities` so the caller can retry without formatting. +fn send_message( + chat_id: i64, + text: &str, + reply_to_message_id: i64, + parse_mode: Option<&str>, +) -> Result { + let mut payload = serde_json::json!({ + "chat_id": chat_id, + "text": text, + "reply_to_message_id": reply_to_message_id, + }); + + if let Some(mode) = parse_mode { + payload["parse_mode"] = serde_json::Value::String(mode.to_string()); + } + + let payload_bytes = serde_json::to_vec(&payload) + .map_err(|e| SendError::Other(format!("Failed to serialize payload: {}", e)))?; + + let headers = serde_json::json!({ "Content-Type": "application/json" }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage", + &headers.to_string(), + Some(&payload_bytes), + None, + ); + + match result { + Ok(http_response) => { + if http_response.status == 400 { + let body_str = String::from_utf8_lossy(&http_response.body); + if body_str.contains("can't parse entities") { + return Err(SendError::ParseEntities(body_str.to_string())); + } + return Err(SendError::Other(format!( + "Telegram API returned 400: {}", + body_str + ))); + } + + if http_response.status != 200 { + let body_str = String::from_utf8_lossy(&http_response.body); + return Err(SendError::Other(format!( + "Telegram API returned status {}: {}", + http_response.status, body_str + ))); + } + + let api_response: TelegramApiResponse = + serde_json::from_slice(&http_response.body) + .map_err(|e| SendError::Other(format!("Failed to parse response: {}", e)))?; + + if !api_response.ok { + return Err(SendError::Other(format!( + "Telegram API error: {}", + api_response + .description + .unwrap_or_else(|| "unknown".to_string()) + ))); + } + + Ok(api_response.result.map(|r| r.message_id).unwrap_or(0)) + } + Err(e) => Err(SendError::Other(format!("HTTP request failed: {}", e))), + } +} + // ============================================================================ // Webhook Management // ============================================================================ @@ -604,6 +731,7 @@ fn delete_webhook() -> Result<(), String> { "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/deleteWebhook", &headers.to_string(), None, + None, ); match result { @@ -666,6 +794,7 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<() "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/setWebhook", &headers.to_string(), Some(&body_bytes), + None, ); match result { @@ -700,6 +829,48 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<() } } +// ============================================================================ +// Pairing Reply +// ============================================================================ + +/// Send a pairing code message to a chat. Used when an unknown user DMs the bot. +fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> { + let payload = serde_json::json!({ + "chat_id": chat_id, + "text": format!( + "To pair with this bot, run: `ironclaw pairing approve telegram {}`", + code + ), + "parse_mode": "Markdown", + }); + + let payload_bytes = serde_json::to_vec(&payload) + .map_err(|e| format!("Failed to serialize payload: {}", e))?; + + let headers = serde_json::json!({ + "Content-Type": "application/json" + }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage", + &headers.to_string(), + Some(&payload_bytes), + None, + ); + + match result { + Ok(response) => { + if response.status != 200 { + let body_str = String::from_utf8_lossy(&response.body); + return Err(format!("HTTP {}: {}", response.status, body_str)); + } + Ok(()) + } + Err(e) => Err(format!("HTTP request failed: {}", e)), + } +} + // ============================================================================ // Update Handling // ============================================================================ @@ -719,11 +890,16 @@ fn handle_update(update: TelegramUpdate) { /// Process a single message. fn handle_message(message: TelegramMessage) { - // Skip messages without text - let text = match message.text { - Some(t) if !t.is_empty() => t, - _ => return, - }; + // Use text or caption (for media messages) + let content = message + .text + .filter(|t| !t.is_empty()) + .or_else(|| message.caption.filter(|c| !c.is_empty())) + .unwrap_or_default(); + + if content.is_empty() { + return; + } // Skip messages without a sender (channel posts) let from = match message.from { @@ -736,41 +912,111 @@ fn handle_message(message: TelegramMessage) { return; } - // Owner validation: silently drop messages from non-owner users - if let Some(owner_id_str) = channel_host::workspace_read(OWNER_ID_PATH) { - if !owner_id_str.is_empty() { - if let Ok(owner_id) = owner_id_str.parse::() { - if from.id != owner_id { - channel_host::log( - channel_host::LogLevel::Debug, - &format!( - "Dropping message from non-owner user {} (owner: {})", - from.id, owner_id - ), - ); - return; + let is_private = message.chat.chat_type == "private"; + + // Owner validation: when owner_id is set, only that user can message + let owner_configured = channel_host::workspace_read(OWNER_ID_PATH) + .map(|s| !s.is_empty()) + .unwrap_or(false); + + if owner_configured { + if let Ok(owner_id) = channel_host::workspace_read(OWNER_ID_PATH) + .unwrap() + .parse::() + { + if from.id != owner_id { + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Dropping message from non-owner user {} (owner: {})", + from.id, owner_id + ), + ); + return; + } + } + } else if is_private { + // No owner_id: apply dm_policy for private chats + let dm_policy = channel_host::workspace_read(DM_POLICY_PATH) + .unwrap_or_else(|| "pairing".to_string()); + + if dm_policy != "open" { + // Build effective allow list: config allow_from + pairing store + let mut allowed: Vec = channel_host::workspace_read(ALLOW_FROM_PATH) + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) { + allowed.extend(store_allowed); + } + + let id_str = from.id.to_string(); + let username_opt = from.username.as_deref(); + let is_allowed = allowed.contains(&"*".to_string()) + || allowed.contains(&id_str) + || username_opt.map_or(false, |u| allowed.contains(&u.to_string())); + + if !is_allowed { + if dm_policy == "pairing" { + // Upsert pairing request and send reply + let meta = serde_json::json!({ + "chat_id": message.chat.id, + "user_id": from.id, + "username": username_opt, + }) + .to_string(); + + match channel_host::pairing_upsert_request(CHANNEL_NAME, &id_str, &meta) { + Ok(result) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Pairing request for user {} (chat {}): code {}", + from.id, message.chat.id, result.code + ), + ); + if result.created { + let _ = send_pairing_reply(message.chat.id, &result.code); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Pairing upsert failed: {}", e), + ); + } + } } + return; } } } - let is_private = message.chat.chat_type == "private"; - - // For group chats, check if the bot was mentioned - // TODO: Read bot_username from config and check mentions - // For now, process all messages in private chats and groups + // For group chats, only respond if bot was mentioned or respond_to_all is enabled if !is_private { - // In groups, only respond if there's a bot mention or command - // This is a simplified check - proper implementation would use entities - let has_command = text.starts_with('/'); - let has_mention = text.contains('@'); + let respond_to_all = channel_host::workspace_read(RESPOND_TO_ALL_GROUP_PATH) + .as_deref() + .unwrap_or("false") + == "true"; - if !has_command && !has_mention { - channel_host::log( - channel_host::LogLevel::Debug, - &format!("Ignoring group message without mention: {}", text), - ); - return; + if !respond_to_all { + let has_command = content.starts_with('/'); + let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH) + .unwrap_or_default(); + let has_bot_mention = if bot_username.is_empty() { + content.contains('@') + } else { + let mention = format!("@{}", bot_username); + content.to_lowercase().contains(&mention.to_lowercase()) + }; + + if !has_command && !has_bot_mention { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Ignoring group message without mention: {}", content), + ); + return; + } } } @@ -792,17 +1038,30 @@ fn handle_message(message: TelegramMessage) { let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string()); // Clean the message text (strip bot mentions and commands) - let cleaned_text = clean_message_text(&text); + let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH).unwrap_or_default(); + let cleaned_text = clean_message_text( + &content, + if bot_username.is_empty() { + None + } else { + Some(bot_username.as_str()) + }, + ); - if cleaned_text.is_empty() { + // For /start with no args, emit placeholder so agent can respond with welcome + let content_to_emit = if cleaned_text.is_empty() && content.trim().starts_with('/') { + "[User started the bot]".to_string() + } else if cleaned_text.is_empty() { return; - } + } else { + cleaned_text + }; // Emit the message to the agent channel_host::emit_message(&EmittedMessage { user_id: from.id.to_string(), user_name: Some(user_name), - content: cleaned_text, + content: content_to_emit, thread_id: None, // Telegram doesn't have threads in the same way metadata_json, }); @@ -817,7 +1076,8 @@ fn handle_message(message: TelegramMessage) { } /// Clean message text by removing bot commands and @mentions at the start. -fn clean_message_text(text: &str) -> String { +/// When bot_username is set, only strips that specific mention; otherwise strips any leading @mention. +fn clean_message_text(text: &str, bot_username: Option<&str>) -> String { let mut result = text.trim().to_string(); // Remove leading /command @@ -832,11 +1092,30 @@ fn clean_message_text(text: &str) -> String { // Remove leading @mention if result.starts_with('@') { - if let Some(space_idx) = result.find(' ') { - result = result[space_idx..].trim_start().to_string(); + if let Some(bot) = bot_username { + let mention = format!("@{}", bot); + let mention_lower = mention.to_lowercase(); + let result_lower = result.to_lowercase(); + if result_lower.starts_with(&mention_lower) { + let rest = result[mention.len()..].trim_start(); + if rest.is_empty() { + return String::new(); + } + result = rest.to_string(); + } else if let Some(space_idx) = result.find(' ') { + // Different leading @mention - only strip if it's the bot + let first_word = &result[..space_idx]; + if first_word.eq_ignore_ascii_case(&mention) { + result = result[space_idx..].trim_start().to_string(); + } + } } else { - // Just a mention with no text - return String::new(); + // No bot_username: strip any leading @mention + if let Some(space_idx) = result.find(' ') { + result = result[space_idx..].trim_start().to_string(); + } else { + return String::new(); + } } } @@ -872,12 +1151,22 @@ mod tests { #[test] fn test_clean_message_text() { - assert_eq!(clean_message_text("/start hello"), "hello"); - assert_eq!(clean_message_text("@bot hello world"), "hello world"); - assert_eq!(clean_message_text("/start"), ""); - assert_eq!(clean_message_text("@botname"), ""); - assert_eq!(clean_message_text("just text"), "just text"); - assert_eq!(clean_message_text(" spaced "), "spaced"); + // Without bot_username: strips any leading @mention + assert_eq!(clean_message_text("/start hello", None), "hello"); + assert_eq!(clean_message_text("@bot hello world", None), "hello world"); + assert_eq!(clean_message_text("/start", None), ""); + assert_eq!(clean_message_text("@botname", None), ""); + assert_eq!(clean_message_text("just text", None), "just text"); + assert_eq!(clean_message_text(" spaced ", None), "spaced"); + + // With bot_username: only strips @MyBot, not @alice + assert_eq!(clean_message_text("@MyBot hello", Some("MyBot")), "hello"); + assert_eq!(clean_message_text("@mybot hi", Some("MyBot")), "hi"); + assert_eq!( + clean_message_text("@alice hello", Some("MyBot")), + "@alice hello" + ); + assert_eq!(clean_message_text("@MyBot", Some("MyBot")), ""); } #[test] @@ -945,4 +1234,17 @@ mod tests { assert_eq!(from.id, 789); assert_eq!(from.first_name, "John"); } + + #[test] + fn test_parse_message_with_caption() { + let json = r#"{ + "message_id": 1, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "caption": "What's in this image?" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + assert_eq!(msg.text, None); + assert_eq!(msg.caption.as_deref(), Some("What's in this image?")); + } } diff --git a/channels-src/telegram/telegram.capabilities.json b/channels-src/telegram/telegram.capabilities.json index 70f56e01..41735b52 100644 --- a/channels-src/telegram/telegram.capabilities.json +++ b/channels-src/telegram/telegram.capabilities.json @@ -1,43 +1 @@ -{ - "type": "channel", - "name": "telegram", - "description": "Telegram Bot API channel for receiving and responding to Telegram messages", - "capabilities": { - "http": { - "allowlist": [ - { "host": "api.telegram.org", "path_prefix": "/bot" } - ], - "credentials": { - "telegram_bot": { - "secret_name": "telegram_bot_token", - "location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" }, - "host_patterns": ["api.telegram.org"] - } - }, - "rate_limit": { - "requests_per_minute": 30, - "requests_per_hour": 1000 - } - }, - "secrets": { - "allowed_names": ["telegram_*"] - }, - "channel": { - "allowed_paths": ["/webhook/telegram"], - "allow_polling": true, - "min_poll_interval_ms": 30000, - "workspace_prefix": "channels/telegram/", - "emit_rate_limit": { - "messages_per_minute": 100, - "messages_per_hour": 5000 - } - } - }, - "config": { - "bot_username": null, - "owner_id": null, - "respond_to_all_group_messages": false, - "polling_enabled": false, - "poll_interval_ms": 30000 - } -} +{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}} diff --git a/channels-src/telegram/telegram.wasm b/channels-src/telegram/telegram.wasm deleted file mode 100644 index 14791860..00000000 Binary files a/channels-src/telegram/telegram.wasm and /dev/null differ diff --git a/channels-src/whatsapp/src/lib.rs b/channels-src/whatsapp/src/lib.rs index e28340b8..27d79e2c 100644 --- a/channels-src/whatsapp/src/lib.rs +++ b/channels-src/whatsapp/src/lib.rs @@ -361,6 +361,7 @@ impl Guest for WhatsAppChannel { &api_url, &headers.to_string(), Some(&payload_bytes), + None, ); match result { diff --git a/deploy/cloud-sql-proxy.service b/deploy/cloud-sql-proxy.service new file mode 100644 index 00000000..346a4e72 --- /dev/null +++ b/deploy/cloud-sql-proxy.service @@ -0,0 +1,13 @@ +[Unit] +Description=Cloud SQL Auth Proxy +After=network.target + +[Service] +Type=simple +DynamicUser=yes +ExecStart=/usr/local/bin/cloud-sql-proxy ironclaw-prod:us-central1:ironclaw-db --port=5432 +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/env.example b/deploy/env.example new file mode 100644 index 00000000..1c7dac9e --- /dev/null +++ b/deploy/env.example @@ -0,0 +1,27 @@ +# WARNING: Replace all CHANGE_ME values before deploying. +# Do not use placeholder passwords in production. +DATABASE_URL=postgres://ironclaw:CHANGE_ME@localhost:5432/ironclaw + +# NEAR AI +NEARAI_SESSION_TOKEN=CHANGE_ME +NEARAI_MODEL=claude-3-5-sonnet-20241022 +NEARAI_BASE_URL=https://cloud-api.near.ai +NEARAI_AUTH_URL=https://private.near.ai +NEARAI_API_MODE=chat_completions + +# Agent +AGENT_NAME=ironclaw +CLI_ENABLED=false + +# Web Gateway +GATEWAY_ENABLED=true +# 0.0.0.0 binds to all interfaces (required for Docker --network=host). +# Use 127.0.0.1 if running outside Docker or for local-only access. +GATEWAY_HOST=0.0.0.0 +GATEWAY_PORT=3000 +GATEWAY_AUTH_TOKEN=CHANGE_ME + +# Disabled for initial deploy +SANDBOX_ENABLED=false +HEARTBEAT_ENABLED=false +EMBEDDING_ENABLED=false diff --git a/deploy/ironclaw.service b/deploy/ironclaw.service new file mode 100644 index 00000000..b5aa0a4e --- /dev/null +++ b/deploy/ironclaw.service @@ -0,0 +1,20 @@ +[Unit] +Description=IronClaw AI Assistant +After=cloud-sql-proxy.service docker.service +Requires=cloud-sql-proxy.service + +[Service] +Type=simple +ExecStartPre=/usr/bin/docker pull us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest +ExecStart=/usr/bin/docker run --rm \ + --name ironclaw \ + --env-file /opt/ironclaw/.env \ + --network=host \ + us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest \ + --no-onboard +ExecStop=/usr/bin/docker stop ironclaw +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/setup.sh b/deploy/setup.sh new file mode 100755 index 00000000..0bec03a0 --- /dev/null +++ b/deploy/setup.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# VM bootstrap script for IronClaw on GCP Compute Engine. +# +# Run on a fresh Debian 12 VM after SSH: +# sudo bash setup.sh +# +# Prerequisites: +# - VM has the ironclaw-vm service account attached +# - Cloud SQL Auth Proxy accessible via IAM +# - Artifact Registry image pushed + +set -euo pipefail + +# Must run as root +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: This script must be run as root (sudo bash setup.sh)" + exit 1 +fi + +echo "==> Installing Docker" +apt-get update +apt-get install -y docker.io +systemctl enable docker +systemctl start docker + +echo "==> Installing Cloud SQL Auth Proxy" +curl -fsSL -o /usr/local/bin/cloud-sql-proxy \ + https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.14.3/cloud-sql-proxy.linux.amd64 +chmod +x /usr/local/bin/cloud-sql-proxy + +echo "==> Installing systemd services" +cp /tmp/deploy/cloud-sql-proxy.service /etc/systemd/system/ +cp /tmp/deploy/ironclaw.service /etc/systemd/system/ +systemctl daemon-reload + +echo "==> Starting Cloud SQL Auth Proxy" +systemctl enable cloud-sql-proxy +systemctl start cloud-sql-proxy + +echo "==> Configuring Docker registry auth" +# The VM service account provides Artifact Registry access +gcloud auth configure-docker us-central1-docker.pkg.dev --quiet + +echo "==> Creating config directory" +# Owned by root, readable only by root. Docker reads --env-file as root +# before dropping to uid 1000 (ironclaw) inside the container. +mkdir -p /opt/ironclaw +chmod 700 /opt/ironclaw + +if [ ! -f /opt/ironclaw/.env ]; then + echo "WARNING: /opt/ironclaw/.env does not exist." + echo "Create it with your configuration before starting IronClaw." + echo "See deploy/env.example for the required variables." + echo "" + echo "Then run: systemctl enable ironclaw && systemctl start ironclaw" +else + chmod 600 /opt/ironclaw/.env + echo "==> Starting IronClaw" + systemctl enable ironclaw + systemctl start ironclaw +fi + +echo "==> Setup complete" +echo "" +echo "Verify with:" +echo " systemctl status cloud-sql-proxy" +echo " systemctl status ironclaw" +echo " docker logs ironclaw" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..9b82fcdb --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +# Local development only — do NOT use these credentials in production. +services: + postgres: + image: pgvector/pgvector:pg16 + ports: + - "5432:5432" + environment: + POSTGRES_DB: ironclaw + POSTGRES_USER: ironclaw + POSTGRES_PASSWORD: ironclaw # dev-only, change for any non-local deployment + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ironclaw"] + interval: 5s + timeout: 3s + retries: 5 + +volumes: + pgdata: diff --git a/docs/BUILDING_CHANNELS.md b/docs/BUILDING_CHANNELS.md index a819bc01..4fad5756 100644 --- a/docs/BUILDING_CHANNELS.md +++ b/docs/BUILDING_CHANNELS.md @@ -246,13 +246,46 @@ Create `my-channel.capabilities.json`: ## Building and Deploying +### Supply Chain Security: No Committed Binaries + +**Do not commit compiled WASM binaries.** They are a supply chain risk — the binary in a PR may not match the source. IronClaw builds channels from source: + +- `cargo build` automatically builds `telegram.wasm` via `build.rs` +- The built binary is in `.gitignore` and is not committed +- CI should run `cargo build` (or `./scripts/build-all.sh`) to produce releases + +**Reproducible build:** +```bash +cargo build --release +``` + +Prerequisites: `rustup target add wasm32-wasip2`, `cargo install wasm-tools` (optional; fallback copies raw WASM if unavailable). + +### Telegram Channel (Manual Build) + +```bash +# Add WASM target if needed +rustup target add wasm32-wasip2 + +# Build Telegram channel +./channels-src/telegram/build.sh + +# Install (or use ironclaw onboard to install bundled channel) +mkdir -p ~/.ironclaw/channels +cp channels-src/telegram/telegram.wasm channels-src/telegram/telegram.capabilities.json ~/.ironclaw/channels/ +``` + +**Note**: The main IronClaw binary bundles `telegram.wasm` via `include_bytes!`. When modifying the Telegram channel source, run `./channels-src/telegram/build.sh` **before** building the main crate, so the updated WASM is included. + +### Other Channels + ```bash # Build the WASM component -cd channels/my-channel -cargo component build --release +cd channels-src/my-channel +cargo build --release --target wasm32-wasip2 # Deploy to ~/.ironclaw/channels/ -cp target/wasm32-wasip1/release/my_channel.wasm ~/.ironclaw/channels/my-channel.wasm +cp target/wasm32-wasip2/release/my_channel.wasm ~/.ironclaw/channels/my-channel.wasm cp my-channel.capabilities.json ~/.ironclaw/channels/ ``` diff --git a/docs/TELEGRAM_SETUP.md b/docs/TELEGRAM_SETUP.md new file mode 100644 index 00000000..f9ec24eb --- /dev/null +++ b/docs/TELEGRAM_SETUP.md @@ -0,0 +1,135 @@ +# Telegram Channel Setup + +This guide covers configuring the Telegram channel for IronClaw, including DM pairing for access control. + +## Overview + +The Telegram channel lets you interact with IronClaw via Telegram DMs and groups. It supports: + +- **Webhook mode** (recommended): Instant delivery via tunnel +- **Polling mode**: No tunnel required; ~30s delay +- **DM pairing**: Approve unknown users before they can message the agent +- **Group mentions**: `@YourBot` or `/command` to trigger in groups + +## Prerequisites + +- IronClaw installed and configured (`ironclaw onboard`) +- A Telegram bot token from [@BotFather](https://t.me/BotFather) + +## Quick Start + +### 1. Create a Bot + +1. Message [@BotFather](https://t.me/BotFather) on Telegram +2. Send `/newbot` and follow the prompts +3. Copy the bot token (e.g., `123456789:ABCdefGHIjklMNOpqrsTUVwxyz`) + +### 2. Configure via Setup Wizard + +```bash +ironclaw onboard +``` + +When prompted, enable the Telegram channel and paste your bot token. The wizard will: + +- Validate the token +- Optionally configure a webhook secret +- Set up tunnel (if you want webhook mode) + +### 3. (Optional) Configure Tunnel for Webhooks + +For instant message delivery, expose your agent via a tunnel: + +```bash +# ngrok +ngrok http 8080 + +# Cloudflare +cloudflared tunnel --url http://localhost:8080 +``` + +Set the tunnel URL in settings or via `TUNNEL_URL` env var. Without a tunnel, the channel uses polling (~30s delay). + +## DM Pairing + +When an unknown user DMs your bot, they receive a pairing code. You must approve them before they can message the agent. + +### Flow + +1. Unknown user sends a message to your bot +2. Bot replies: `To pair with this bot, run: ironclaw pairing approve telegram ABC12345` +3. You run: `ironclaw pairing approve telegram ABC12345` +4. User is added to the allow list; future messages are delivered + +### Commands + +```bash +# List pending pairing requests +ironclaw pairing list telegram + +# List as JSON +ironclaw pairing list telegram --json + +# Approve a user by code +ironclaw pairing approve telegram ABC12345 +``` + +### Configuration + +Edit `~/.ironclaw/channels/telegram.capabilities.json` (or the config injected by the host): + +| Option | Values | Default | Description | +|--------|--------|---------|-------------| +| `dm_policy` | `open`, `allowlist`, `pairing` | `pairing` | `open` = allow all; `allowlist` = config + approved only; `pairing` = allowlist + send pairing reply to unknown | +| `allow_from` | `["user_id", "username", "*"]` | `[]` | Pre-approved IDs/usernames. `*` allows everyone. | +| `owner_id` | Telegram user ID | `null` | When set, only this user can message (overrides dm_policy) | +| `bot_username` | Bot username (no @) | `null` | Used for mention detection in groups; when set, only strips this mention from messages | +| `respond_to_all_group_messages` | `true`/`false` | `false` | When true, respond to all group messages; when false, only @mentions and /commands | + +## Manual Installation + +If the channel isn't installed via the wizard: + +```bash +# Build the Telegram channel (requires wasm32-wasip2 target) +rustup target add wasm32-wasip2 +./channels-src/telegram/build.sh + +# Install +mkdir -p ~/.ironclaw/channels +cp channels-src/telegram/telegram.wasm channels-src/telegram/telegram.capabilities.json ~/.ironclaw/channels/ +``` + +## Secrets + +The channel expects a secret named `telegram_bot_token`. Configure via: + +- **Setup wizard**: Saves to encrypted secrets store +- **Environment**: `TELEGRAM_BOT_TOKEN=your_token` +- **Secrets store**: `ironclaw` CLI (if available) + +## Webhook Secret (Optional) + +For webhook validation, set `telegram_webhook_secret` in secrets. Telegram will send `X-Telegram-Bot-Api-Secret-Token` with each request; the host validates it before forwarding. + +## Troubleshooting + +### Messages not delivered + +- **Polling mode**: Check logs for `getUpdates` errors. Ensure the bot token is valid. +- **Webhook mode**: Verify tunnel is running and `TUNNEL_URL` is correct. Telegram requires HTTPS. + +### Pairing code not received + +- Verify the channel can send messages (HTTP allowlist includes `api.telegram.org`) +- Check `dm_policy` is `pairing` (not `allowlist` which blocks without reply) + +### Group mentions not working + +- Set `bot_username` in config to your bot's username (e.g., `MyIronClawBot`) +- Ensure the message contains `@YourBot` or starts with `/` + +### "Connection refused" when starting + +- For webhook mode: Start your tunnel before `ironclaw run` +- For polling only: No tunnel needed; ignore tunnel-related warnings diff --git a/examples/test_heartbeat.rs b/examples/test_heartbeat.rs new file mode 100644 index 00000000..fcb9333d --- /dev/null +++ b/examples/test_heartbeat.rs @@ -0,0 +1,121 @@ +//! Standalone heartbeat test. +//! +//! Exercises the heartbeat system in isolation: connects to the real +//! database, reads the real HEARTBEAT.md, calls the real LLM, and prints +//! every step so you can see exactly where it breaks. +//! +//! Usage: +//! cargo run --example test_heartbeat + +use std::sync::Arc; + +use ironclaw::{ + agent::HeartbeatRunner, + config::Config, + history::Store, + llm::{SessionConfig, create_llm_provider, create_session_manager}, + workspace::Workspace, +}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Load .env and set up logging + let _ = dotenvy::dotenv(); + tracing_subscriber::fmt() + .with_env_filter("ironclaw=debug") + .init(); + + println!("=== Heartbeat Integration Test ===\n"); + + // 1. Load config + let config = Config::from_env() + .await + .map_err(|e| anyhow::anyhow!("Config: {}", e))?; + println!("[1/6] Config loaded"); + println!(" heartbeat.enabled = {}", config.heartbeat.enabled); + println!( + " heartbeat.interval_secs = {}", + config.heartbeat.interval_secs + ); + println!( + " heartbeat.notify_channel = {:?}", + config.heartbeat.notify_channel + ); + println!( + " heartbeat.notify_user = {:?}", + config.heartbeat.notify_user + ); + + // 2. Connect to database + let store = Store::new(&config.database).await?; + store.run_migrations().await?; + println!("[2/6] Database connected"); + + // 3. Create workspace + let workspace = Arc::new(Workspace::new("default", store.pool())); + println!("[3/6] Workspace created"); + + // 4. Read HEARTBEAT.md + let checklist = workspace.heartbeat_checklist().await; + match &checklist { + Ok(Some(content)) => { + let preview: String = content.chars().take(200).collect(); + println!("[4/6] HEARTBEAT.md found ({} chars)", content.len()); + println!(" Preview: {}...", preview); + } + Ok(None) => { + println!("[4/6] HEARTBEAT.md is None (no file, no seed fallback)"); + println!(" Heartbeat will return Skipped."); + } + Err(e) => { + println!("[4/6] HEARTBEAT.md read error: {}", e); + } + } + + // Check if the checklist would be considered "effectively empty" + if let Ok(Some(_)) = checklist { + println!(" (Will verify via runner below)"); + } + + // 5. Create LLM provider + let session = create_session_manager(SessionConfig { + auth_base_url: config.llm.nearai.auth_base_url.clone(), + session_path: config.llm.nearai.session_path.clone(), + }) + .await; + let llm = create_llm_provider(&config.llm, session)?; + println!("[5/6] LLM provider created (model: {})", llm.model_name()); + + // 6. Run heartbeat check + println!("[6/6] Running check_heartbeat()...\n"); + + let hb_config = ironclaw::agent::HeartbeatConfig::default(); + let runner = HeartbeatRunner::new(hb_config, workspace, llm); + + let result = runner.check_heartbeat().await; + + println!("=== Result ===\n"); + match &result { + ironclaw::agent::HeartbeatResult::Ok => { + println!("HeartbeatResult::Ok"); + println!(" LLM responded HEARTBEAT_OK, nothing needs attention."); + } + ironclaw::agent::HeartbeatResult::NeedsAttention(msg) => { + println!("HeartbeatResult::NeedsAttention"); + println!(" Message:\n{}", msg); + } + ironclaw::agent::HeartbeatResult::Skipped => { + println!("HeartbeatResult::Skipped"); + println!(" No checklist found, or checklist was effectively empty."); + println!(" This means the HEARTBEAT.md either:"); + println!(" - Does not exist in the workspace database"); + println!(" - Contains only headers, comments, and empty checkboxes"); + } + ironclaw::agent::HeartbeatResult::Failed(err) => { + println!("HeartbeatResult::Failed"); + println!(" Error: {}", err); + } + } + + Ok(()) +} diff --git a/migrations/V4__sandbox_columns.sql b/migrations/V4__sandbox_columns.sql new file mode 100644 index 00000000..7510e847 --- /dev/null +++ b/migrations/V4__sandbox_columns.sql @@ -0,0 +1,10 @@ +-- Add project_dir and user_id columns for sandbox job tracking. +-- user_id was previously hardcoded to "default" in the Rust layer; +-- now it's persisted so we can filter per-user. + +ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS project_dir TEXT; +ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT 'default'; + +CREATE INDEX IF NOT EXISTS idx_agent_jobs_source ON agent_jobs(source); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_user ON agent_jobs(user_id); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_created ON agent_jobs(created_at DESC); diff --git a/migrations/V5__claude_code.sql b/migrations/V5__claude_code.sql new file mode 100644 index 00000000..f0a20426 --- /dev/null +++ b/migrations/V5__claude_code.sql @@ -0,0 +1,14 @@ +-- Track which mode a sandbox job uses (worker vs claude_code). +ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS job_mode TEXT NOT NULL DEFAULT 'worker'; + +-- Persist Claude Code streaming events so they survive restarts and can be +-- loaded when the frontend opens a job detail view after the fact. +CREATE TABLE IF NOT EXISTS claude_code_events ( + id BIGSERIAL PRIMARY KEY, + job_id UUID NOT NULL REFERENCES agent_jobs(id), + event_type TEXT NOT NULL, + data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_cc_events_job ON claude_code_events(job_id, id); diff --git a/migrations/V6__routines.sql b/migrations/V6__routines.sql new file mode 100644 index 00000000..36f63cb2 --- /dev/null +++ b/migrations/V6__routines.sql @@ -0,0 +1,73 @@ +-- Routines: scheduled and reactive job system. +-- +-- A routine is a named, persistent, user-owned task with a trigger and an action. +-- Triggers fire independently (cron, event, webhook, manual) so only the +-- relevant routine's prompt hits the LLM, not the whole checklist. + +CREATE TABLE routines ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + user_id TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + + -- Trigger definition + trigger_type TEXT NOT NULL, -- 'cron', 'event', 'webhook', 'manual' + trigger_config JSONB NOT NULL, -- type-specific config (schedule, pattern, etc.) + + -- Action definition + action_type TEXT NOT NULL, -- 'lightweight', 'full_job' + action_config JSONB NOT NULL, -- prompt, context_paths, max_tokens / title, max_iterations + + -- Guardrails + cooldown_secs INTEGER NOT NULL DEFAULT 300, + max_concurrent INTEGER NOT NULL DEFAULT 1, + dedup_window_secs INTEGER, -- NULL = no dedup + + -- Notification preferences + notify_channel TEXT, -- NULL = use default + notify_user TEXT NOT NULL DEFAULT 'default', + notify_on_success BOOLEAN NOT NULL DEFAULT false, + notify_on_failure BOOLEAN NOT NULL DEFAULT true, + notify_on_attention BOOLEAN NOT NULL DEFAULT true, + + -- Runtime state (updated by engine) + state JSONB NOT NULL DEFAULT '{}', + last_run_at TIMESTAMPTZ, + next_fire_at TIMESTAMPTZ, -- pre-computed for cron triggers + run_count BIGINT NOT NULL DEFAULT 0, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + UNIQUE (user_id, name) +); + +-- Fast lookup: "which cron routines need to fire right now?" +CREATE INDEX idx_routines_next_fire + ON routines (next_fire_at) + WHERE enabled AND next_fire_at IS NOT NULL; + +-- Fast lookup: event triggers for a user +CREATE INDEX idx_routines_event_triggers + ON routines (user_id) + WHERE enabled AND trigger_type = 'event'; + +-- Audit log of individual routine executions. +CREATE TABLE routine_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + routine_id UUID NOT NULL REFERENCES routines(id) ON DELETE CASCADE, + trigger_type TEXT NOT NULL, + trigger_detail TEXT, -- e.g. matched message preview, cron expression + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + status TEXT NOT NULL DEFAULT 'running', -- running, ok, attention, failed + result_summary TEXT, + tokens_used INTEGER, + job_id UUID REFERENCES agent_jobs(id), -- non-NULL for full_job runs + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_routine_runs_routine ON routine_runs (routine_id); +CREATE INDEX idx_routine_runs_status ON routine_runs (status) WHERE status = 'running'; diff --git a/migrations/V7__rename_events.sql b/migrations/V7__rename_events.sql new file mode 100644 index 00000000..3676fbf7 --- /dev/null +++ b/migrations/V7__rename_events.sql @@ -0,0 +1,3 @@ +-- Rename claude_code_events to job_events (generic for all sandbox job types). +ALTER TABLE claude_code_events RENAME TO job_events; +ALTER INDEX idx_cc_events_job RENAME TO idx_job_events_job; diff --git a/migrations/V8__settings.sql b/migrations/V8__settings.sql new file mode 100644 index 00000000..515b0a74 --- /dev/null +++ b/migrations/V8__settings.sql @@ -0,0 +1,16 @@ +-- Settings table: key-value store for all user configuration. +-- +-- Replaces ~/.ironclaw/settings.json, session.json, and mcp-servers.json. +-- Keys use dotted paths matching the existing Settings.get()/set() convention +-- (e.g., "agent.name", "sandbox.enabled", "mcp_servers"). +-- One row per setting so individual values can be updated atomically. + +CREATE TABLE IF NOT EXISTS settings ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, key) +); + +CREATE INDEX IF NOT EXISTS idx_settings_user ON settings (user_id); diff --git a/release-plz.toml b/release-plz.toml new file mode 100644 index 00000000..e8e0670f --- /dev/null +++ b/release-plz.toml @@ -0,0 +1,2 @@ +[workspace] +git_release_enable = false diff --git a/scripts/build-all.sh b/scripts/build-all.sh new file mode 100755 index 00000000..713940a1 --- /dev/null +++ b/scripts/build-all.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Build IronClaw and all bundled channels. +# +# Run this before release or when channel sources have changed. +# The main binary bundles telegram.wasm via include_bytes!; it must exist. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "Building bundled channels..." +if [ -d "channels-src/telegram" ]; then + ./channels-src/telegram/build.sh +fi + +echo "" +echo "Building IronClaw..." +cargo build --release + +echo "" +echo "Done. Binary: target/release/ironclaw" diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 76cefdf9..044981b8 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -9,25 +9,26 @@ use uuid::Uuid; use crate::agent::compaction::ContextCompactor; use crate::agent::context_monitor::ContextMonitor; use crate::agent::heartbeat::spawn_heartbeat; +use crate::agent::routine_engine::{RoutineEngine, spawn_cron_ticker}; use crate::agent::self_repair::{DefaultSelfRepair, RepairResult, SelfRepair}; use crate::agent::session::{PendingApproval, Session, ThreadState}; use crate::agent::session_manager::SessionManager; use crate::agent::submission::{Submission, SubmissionParser, SubmissionResult}; use crate::agent::{HeartbeatConfig as AgentHeartbeatConfig, MessageIntent, Router, Scheduler}; use crate::channels::{ChannelManager, IncomingMessage, OutgoingResponse, StatusUpdate}; -use crate::config::{AgentConfig, HeartbeatConfig}; +use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig}; use crate::context::ContextManager; use crate::context::JobContext; +use crate::db::Database; use crate::error::Error; use crate::extensions::ExtensionManager; -use crate::history::Store; use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult}; use crate::safety::SafetyLayer; use crate::tools::ToolRegistry; use crate::workspace::Workspace; /// Collapse a tool output string into a single-line preview for display. -fn truncate_for_preview(output: &str, max_chars: usize) -> String { +pub(crate) fn truncate_for_preview(output: &str, max_chars: usize) -> String { let collapsed: String = output .chars() .take(max_chars + 50) @@ -36,8 +37,14 @@ fn truncate_for_preview(output: &str, max_chars: usize) -> String { .split_whitespace() .collect::>() .join(" "); - if collapsed.len() > max_chars { - format!("{}...", &collapsed[..max_chars]) + // char_indices gives us byte offsets at char boundaries, so the slice is always valid UTF-8. + if collapsed.chars().count() > max_chars { + let byte_offset = collapsed + .char_indices() + .nth(max_chars) + .map(|(i, _)| i) + .unwrap_or(collapsed.len()); + format!("{}...", &collapsed[..byte_offset]) } else { collapsed } @@ -58,7 +65,7 @@ enum AgenticLoopResult { /// /// Bundles the shared components to reduce argument count. pub struct AgentDeps { - pub store: Option>, + pub store: Option>, pub llm: Arc, pub safety: Arc, pub tools: Arc, @@ -77,6 +84,7 @@ pub struct Agent { session_manager: Arc, context_monitor: ContextMonitor, heartbeat_config: Option, + routine_config: Option, } impl Agent { @@ -89,6 +97,7 @@ impl Agent { deps: AgentDeps, channels: ChannelManager, heartbeat_config: Option, + routine_config: Option, context_manager: Option>, session_manager: Option>, ) -> Self { @@ -116,11 +125,12 @@ impl Agent { session_manager, context_monitor: ContextMonitor::new(), heartbeat_config, + routine_config, } } // Convenience accessors - fn store(&self) -> Option<&Arc> { + fn store(&self) -> Option<&Arc> { self.deps.store.as_ref() } @@ -256,53 +266,28 @@ impl Agent { let channels = self.channels.clone(); tokio::spawn(async move { while let Some(response) = notify_rx.recv().await { - // Route notification to configured channel/user, or broadcast to all - match (¬ify_channel, ¬ify_user) { - (Some(channel), Some(user)) => { - // Send to specific channel and user - if let Err(e) = - channels.broadcast(channel, user, response.clone()).await - { + let user = notify_user.as_deref().unwrap_or("default"); + + // Try the configured channel first, fall back to + // broadcasting on all channels. + let targeted_ok = if let Some(ref channel) = notify_channel { + channels + .broadcast(channel, user, response.clone()) + .await + .is_ok() + } else { + false + }; + + if !targeted_ok { + let results = channels.broadcast_all(user, response).await; + for (ch, result) in results { + if let Err(e) = result { tracing::warn!( - "Failed to send heartbeat to {}/{}: {}", - channel, - user, + "Failed to broadcast heartbeat to {}: {}", + ch, e ); - } else { - tracing::debug!( - "Heartbeat notification sent to {}/{}", - channel, - user - ); - } - } - (None, Some(user)) => { - // Broadcast to all channels for this user - let results = channels.broadcast_all(user, response).await; - for (ch, result) in results { - if let Err(e) = result { - tracing::warn!( - "Failed to broadcast heartbeat to {}: {}", - ch, - e - ); - } - } - } - _ => { - // No explicit target, broadcast to all channels - // for the default user so notifications actually - // reach someone instead of vanishing into logs. - let results = channels.broadcast_all("default", response).await; - for (ch, result) in results { - if let Err(e) = result { - tracing::warn!( - "Failed to broadcast heartbeat to {}: {}", - ch, - e - ); - } } } } @@ -330,6 +315,85 @@ impl Agent { None }; + // Spawn routine engine if enabled + let routine_handle = if let Some(ref rt_config) = self.routine_config { + if rt_config.enabled { + if let (Some(store), Some(workspace)) = (self.store(), self.workspace()) { + // Set up notification channel (same pattern as heartbeat) + let (notify_tx, mut notify_rx) = + tokio::sync::mpsc::channel::(32); + + let engine = Arc::new(RoutineEngine::new( + rt_config.clone(), + Arc::clone(store), + self.llm().clone(), + Arc::clone(workspace), + notify_tx, + )); + + // Register routine tools + self.deps + .tools + .register_routine_tools(Arc::clone(store), Arc::clone(&engine)); + + // Load initial event cache + engine.refresh_event_cache().await; + + // Spawn notification forwarder + let channels = self.channels.clone(); + tokio::spawn(async move { + while let Some(response) = notify_rx.recv().await { + let user = response + .metadata + .get("notify_user") + .and_then(|v| v.as_str()) + .unwrap_or("default") + .to_string(); + let results = channels.broadcast_all(&user, response).await; + for (ch, result) in results { + if let Err(e) = result { + tracing::warn!( + "Failed to broadcast routine notification to {}: {}", + ch, + e + ); + } + } + } + }); + + // Spawn cron ticker + let cron_interval = + std::time::Duration::from_secs(rt_config.cron_check_interval_secs); + let cron_handle = spawn_cron_ticker(Arc::clone(&engine), cron_interval); + + // Store engine reference for event trigger checking + // Safety: we're in run() which takes self, no other reference exists + let engine_ref = Arc::clone(&engine); + // SAFETY: self is consumed by run(), we can smuggle the engine in + // via a local to use in the message loop below. + + tracing::info!( + "Routines enabled: cron ticker every {}s, max {} concurrent", + rt_config.cron_check_interval_secs, + rt_config.max_concurrent_routines + ); + + Some((cron_handle, engine_ref)) + } else { + tracing::warn!("Routines enabled but store/workspace not available"); + None + } + } else { + None + } + } else { + None + }; + + // Extract engine ref for use in message loop + let routine_engine_for_loop = routine_handle.as_ref().map(|(_, e)| Arc::clone(e)); + // Main message loop tracing::info!("Agent {} ready and listening", self.config.name); @@ -374,6 +438,14 @@ impl Agent { .await; } } + + // Check event triggers (cheap in-memory regex, fires async if matched) + if let Some(ref engine) = routine_engine_for_loop { + let fired = engine.check_event_triggers(&message).await; + if fired > 0 { + tracing::debug!("Fired {} event-triggered routines", fired); + } + } } // Cleanup @@ -383,6 +455,9 @@ impl Agent { if let Some(handle) = heartbeat_handle { handle.abort(); } + if let Some((cron_handle, _)) = routine_handle { + cron_handle.abort(); + } self.scheduler.stop_all().await; self.channels.shutdown_all().await?; @@ -393,6 +468,11 @@ impl Agent { // Parse submission type first let submission = SubmissionParser::parse(&message.content); + // Hydrate thread from DB if it's a historical thread not in memory + if let Some(ref external_thread_id) = message.thread_id { + self.maybe_hydrate_thread(message, external_thread_id).await; + } + // Resolve session and thread let (session, thread_id) = self .session_manager @@ -444,6 +524,9 @@ impl Agent { self.process_user_input(message, session, thread_id, &content) .await } + Submission::SystemCommand { command, args } => { + self.handle_system_command(&command, &args).await + } Submission::Undo => self.process_undo(session, thread_id).await, Submission::Redo => self.process_redo(session, thread_id).await, Submission::Interrupt => self.process_interrupt(session, thread_id).await, @@ -515,6 +598,105 @@ impl Agent { } } + /// Hydrate a historical thread from DB into memory if not already present. + /// + /// Called before `resolve_thread` so that the session manager finds the + /// thread on lookup instead of creating a new one. + /// + /// Creates an in-memory thread with the exact UUID the frontend sent, + /// even when the conversation has zero messages (e.g. a brand-new + /// assistant thread). Without this, `resolve_thread` would mint a + /// fresh UUID and all messages would land in the wrong conversation. + async fn maybe_hydrate_thread(&self, message: &IncomingMessage, external_thread_id: &str) { + // Only hydrate UUID-shaped thread IDs (web gateway uses UUIDs) + let thread_uuid = match Uuid::parse_str(external_thread_id) { + Ok(id) => id, + Err(_) => return, + }; + + // Check if already in memory + let session = self + .session_manager + .get_or_create_session(&message.user_id) + .await; + { + let sess = session.lock().await; + if sess.threads.contains_key(&thread_uuid) { + return; + } + } + + // Load history from DB (may be empty for a newly created thread). + let mut chat_messages: Vec = Vec::new(); + let msg_count; + + if let Some(store) = self.store() { + let db_messages = store + .list_conversation_messages(thread_uuid) + .await + .unwrap_or_default(); + msg_count = db_messages.len(); + chat_messages = db_messages + .iter() + .filter_map(|m| match m.role.as_str() { + "user" => Some(ChatMessage::user(&m.content)), + "assistant" => Some(ChatMessage::assistant(&m.content)), + _ => None, + }) + .collect(); + } else { + msg_count = 0; + } + + // Create thread with the historical ID and restore messages + let session_id = { + let sess = session.lock().await; + sess.id + }; + + let mut thread = crate::agent::session::Thread::with_id(thread_uuid, session_id); + if !chat_messages.is_empty() { + thread.restore_from_messages(chat_messages); + } + + // Restore response chain from conversation metadata + if let Some(store) = self.store() + && let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await + && let Some(rid) = metadata + .get("last_response_id") + .and_then(|v| v.as_str()) + .map(String::from) + { + thread.last_response_id = Some(rid.clone()); + self.llm() + .seed_response_chain(&thread_uuid.to_string(), rid); + tracing::debug!("Restored response chain for thread {}", thread_uuid); + } + + // Insert into session and register with session manager + { + let mut sess = session.lock().await; + sess.threads.insert(thread_uuid, thread); + sess.active_thread = Some(thread_uuid); + sess.last_active_at = chrono::Utc::now(); + } + + self.session_manager + .register_thread( + &message.user_id, + &message.channel, + thread_uuid, + Arc::clone(&session), + ) + .await; + + tracing::debug!( + "Hydrated thread {} from DB ({} messages)", + thread_uuid, + msg_count + ); + } + async fn process_user_input( &self, message: &IncomingMessage, @@ -694,6 +876,7 @@ impl Agent { match result { Ok(AgenticLoopResult::Response(response)) => { thread.complete_turn(&response); + self.persist_response_chain(thread); let _ = self .channels .send_status( @@ -702,6 +885,10 @@ impl Agent { &message.metadata, ) .await; + + // Fire-and-forget: persist turn to DB + self.persist_turn(thread_id, &message.user_id, content, Some(&response)); + Ok(SubmissionResult::response(response)) } Ok(AgenticLoopResult::NeedApproval { pending }) => { @@ -728,11 +915,94 @@ impl Agent { } Err(e) => { thread.fail_turn(e.to_string()); + + // Persist the user message even on failure + self.persist_turn(thread_id, &message.user_id, content, None); + Ok(SubmissionResult::error(e.to_string())) } } } + /// Fire-and-forget: persist a turn (user message + optional assistant response) to the DB. + fn persist_turn( + &self, + thread_id: Uuid, + user_id: &str, + user_input: &str, + response: Option<&str>, + ) { + let store = match self.store() { + Some(s) => Arc::clone(s), + None => return, + }; + + let user_id = user_id.to_string(); + let user_input = user_input.to_string(); + let response = response.map(String::from); + + tokio::spawn(async move { + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", &user_id, None) + .await + { + tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e); + return; + } + + if let Err(e) = store + .add_conversation_message(thread_id, "user", &user_input) + .await + { + tracing::warn!("Failed to persist user message: {}", e); + return; + } + + if let Some(ref resp) = response + && let Err(e) = store + .add_conversation_message(thread_id, "assistant", resp) + .await + { + tracing::warn!("Failed to persist assistant message: {}", e); + } + }); + } + + /// Sync the provider's response chain ID to the thread and DB metadata. + /// + /// Call after a successful agentic loop to persist the latest + /// `previous_response_id` so chaining survives restarts. + fn persist_response_chain(&self, thread: &mut crate::agent::session::Thread) { + let tid = thread.id.to_string(); + let response_id = match self.llm().get_response_chain_id(&tid) { + Some(rid) => rid, + None => return, + }; + + // Update in-memory thread + thread.last_response_id = Some(response_id.clone()); + + // Fire-and-forget DB write + let store = match self.store() { + Some(s) => Arc::clone(s), + None => return, + }; + let thread_id = thread.id; + tokio::spawn(async move { + let val = serde_json::json!(response_id); + if let Err(e) = store + .update_conversation_metadata_field(thread_id, "last_response_id", &val) + .await + { + tracing::warn!( + "Failed to persist response chain for thread {}: {}", + thread_id, + e + ); + } + }); + } + /// Run the agentic loop: call LLM, execute tools, repeat until text response. /// /// Returns `AgenticLoopResult::Response` on completion, or @@ -782,7 +1052,7 @@ impl Agent { iteration += 1; if iteration > MAX_TOOL_ITERATIONS { return Err(crate::error::LlmError::InvalidResponse { - provider: "nearai".to_string(), + provider: "agent".to_string(), reason: format!("Exceeded maximum tool iterations ({})", MAX_TOOL_ITERATIONS), } .into()); @@ -791,14 +1061,14 @@ impl Agent { // Check if interrupted { let sess = session.lock().await; - if let Some(thread) = sess.threads.get(&thread_id) { - if thread.state == ThreadState::Interrupted { - return Err(crate::error::JobError::ContextError { - id: thread_id, - reason: "Interrupted".to_string(), - } - .into()); + if let Some(thread) = sess.threads.get(&thread_id) + && thread.state == ThreadState::Interrupted + { + return Err(crate::error::JobError::ContextError { + id: thread_id, + reason: "Interrupted".to_string(), } + .into()); } } @@ -808,11 +1078,23 @@ impl Agent { // Call LLM with current context let context = ReasoningContext::new() .with_messages(context_messages.clone()) - .with_tools(tool_defs); + .with_tools(tool_defs) + .with_metadata({ + let mut m = std::collections::HashMap::new(); + m.insert("thread_id".to_string(), thread_id.to_string()); + m + }); - let result = reasoning.respond_with_tools(&context).await?; + let output = reasoning.respond_with_tools(&context).await?; - match result { + // Track token usage for budget enforcement + tracing::debug!( + "LLM call used {} input + {} output tokens", + output.usage.input_tokens, + output.usage.output_tokens + ); + + match output.result { RespondResult::Text(text) => { // If no tools have been executed yet, prompt the LLM to use tools // This handles the case where the model explains what it will do @@ -832,13 +1114,16 @@ impl Agent { // Tools have been executed or we've tried multiple times, return response return Ok(AgenticLoopResult::Response(text)); } - RespondResult::ToolCalls(tool_calls) => { + RespondResult::ToolCalls { + tool_calls, + content, + } => { tools_executed = true; // Add the assistant message with tool_calls to context. - // OpenAI-compatible APIs require this before tool-result messages. + // OpenAI protocol requires this before tool-result messages. context_messages.push(ChatMessage::assistant_with_tool_calls( - "", + content, tool_calls.clone(), )); @@ -858,11 +1143,11 @@ impl Agent { // Record tool calls in the thread { let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - if let Some(turn) = thread.last_turn_mut() { - for tc in &tool_calls { - turn.record_tool_call(&tc.name, tc.arguments.clone()); - } + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + for tc in &tool_calls { + turn.record_tool_call(&tc.name, tc.arguments.clone()); } } } @@ -870,27 +1155,56 @@ impl Agent { // Execute each tool (with approval checking) for tc in tool_calls { // Check if tool requires approval - if let Some(tool) = self.tools().get(&tc.name).await { - if tool.requires_approval() { - // Check if auto-approved for this session - let is_auto_approved = { - let sess = session.lock().await; - sess.is_tool_auto_approved(&tc.name) + if let Some(tool) = self.tools().get(&tc.name).await + && tool.requires_approval() + { + // Check if auto-approved for this session + let mut is_auto_approved = { + let sess = session.lock().await; + sess.is_tool_auto_approved(&tc.name) + }; + + // For shell commands, override auto-approval for + // destructive patterns that should always require + // explicit per-invocation approval. + if is_auto_approved + && tc.name == "shell" + && let Some(cmd) = tc + .arguments + .get("command") + .and_then(|c| c.as_str().map(String::from)) + .or_else(|| { + tc.arguments + .as_str() + .and_then(|s| { + serde_json::from_str::(s).ok() + }) + .and_then(|v| { + v.get("command") + .and_then(|c| c.as_str().map(String::from)) + }) + }) + && crate::tools::builtin::shell::requires_explicit_approval(&cmd) + { + tracing::info!( + "Shell command '{}' requires explicit approval despite auto-approve", + cmd.chars().take(80).collect::() + ); + is_auto_approved = false; + } + + if !is_auto_approved { + // Need approval - store pending request and return + let pending = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: tc.name.clone(), + parameters: tc.arguments.clone(), + description: tool.description().to_string(), + tool_call_id: tc.id.clone(), + context_messages: context_messages.clone(), }; - if !is_auto_approved { - // Need approval - store pending request and return - let pending = PendingApproval { - request_id: Uuid::new_v4(), - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - description: tool.description().to_string(), - tool_call_id: tc.id.clone(), - context_messages: context_messages.clone(), - }; - - return Ok(AgenticLoopResult::NeedApproval { pending }); - } + return Ok(AgenticLoopResult::NeedApproval { pending }); } } @@ -921,34 +1235,34 @@ impl Agent { ) .await; - if let Ok(ref output) = tool_result { - if !output.is_empty() { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolResult { - name: tc.name.clone(), - preview: truncate_for_preview(output, 200), - }, - &message.metadata, - ) - .await; - } + if let Ok(ref output) = tool_result + && !output.is_empty() + { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolResult { + name: tc.name.clone(), + preview: output.clone(), + }, + &message.metadata, + ) + .await; } // Record result in thread { let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - if let Some(turn) = thread.last_turn_mut() { - match &tool_result { - Ok(output) => { - turn.record_tool_result(serde_json::json!(output)); - } - Err(e) => { - turn.record_tool_error(e.to_string()); - } + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + match &tool_result { + Ok(output) => { + turn.record_tool_result(serde_json::json!(output)); + } + Err(e) => { + turn.record_tool_error(e.to_string()); } } } @@ -960,10 +1274,26 @@ impl Agent { if let Some((ext_name, instructions)) = detect_auth_awaiting(&tc.name, &tool_result) { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.enter_auth_mode(ext_name); + let auth_data = parse_auth_result(&tool_result); + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) { + thread.enter_auth_mode(ext_name.clone()); + } } + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthRequired { + extension_name: ext_name, + instructions: Some(instructions.clone()), + auth_url: auth_data.auth_url, + setup_url: auth_data.setup_url, + }, + &message.metadata, + ) + .await; return Ok(AgenticLoopResult::Response(instructions)); } @@ -1024,19 +1354,59 @@ impl Agent { .into()); } - // Execute with timeout - let result = tokio::time::timeout(std::time::Duration::from_secs(60), async { + tracing::debug!( + tool = %tool_name, + params = %params, + "Tool call started" + ); + + // Execute with per-tool timeout + let timeout = tool.execution_timeout(); + let start = std::time::Instant::now(); + let result = tokio::time::timeout(timeout, async { tool.execute(params.clone(), job_ctx).await }) - .await - .map_err(|_| crate::error::ToolError::Timeout { - name: tool_name.to_string(), - timeout: std::time::Duration::from_secs(60), - })? - .map_err(|e| crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: e.to_string(), - })?; + .await; + let elapsed = start.elapsed(); + + match &result { + Ok(Ok(output)) => { + let result_str = serde_json::to_string(&output.result) + .unwrap_or_else(|_| "".to_string()); + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + result = %result_str, + "Tool call succeeded" + ); + } + Ok(Err(e)) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + error = %e, + "Tool call failed" + ); + } + Err(_) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + timeout_secs = timeout.as_secs(), + "Tool call timed out" + ); + } + } + + let result = result + .map_err(|_| crate::error::ToolError::Timeout { + name: tool_name.to_string(), + timeout, + })? + .map_err(|e| crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: e.to_string(), + })?; // Convert result to string serde_json::to_string_pretty(&result.result).map_err(|e| { @@ -1275,17 +1645,17 @@ impl Agent { }; // Verify request ID if provided - if let Some(req_id) = request_id { - if req_id != pending.request_id { - // Put it back and return error - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.await_approval(pending); - } - return Ok(SubmissionResult::error( - "Request ID mismatch. Use the correct request ID.", - )); + if let Some(req_id) = request_id + && req_id != pending.request_id + { + // Put it back and return error + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) { + thread.await_approval(pending); } + return Ok(SubmissionResult::error( + "Request ID mismatch. Use the correct request ID.", + )); } if approved { @@ -1339,20 +1709,20 @@ impl Agent { ) .await; - if let Ok(ref output) = tool_result { - if !output.is_empty() { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolResult { - name: pending.tool_name.clone(), - preview: truncate_for_preview(output, 200), - }, - &message.metadata, - ) - .await; - } + if let Ok(ref output) = tool_result + && !output.is_empty() + { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolResult { + name: pending.tool_name.clone(), + preview: output.clone(), + }, + &message.metadata, + ) + .await; } // Build context including the tool result @@ -1361,15 +1731,15 @@ impl Agent { // Record result in thread { let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - if let Some(turn) = thread.last_turn_mut() { - match &tool_result { - Ok(output) => { - turn.record_tool_result(serde_json::json!(output)); - } - Err(e) => { - turn.record_tool_error(e.to_string()); - } + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + match &tool_result { + Ok(output) => { + turn.record_tool_result(serde_json::json!(output)); + } + Err(e) => { + turn.record_tool_error(e.to_string()); } } } @@ -1380,10 +1750,11 @@ impl Agent { if let Some((ext_name, instructions)) = detect_auth_awaiting(&pending.tool_name, &tool_result) { + let auth_data = parse_auth_result(&tool_result); { let mut sess = session.lock().await; if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.enter_auth_mode(ext_name); + thread.enter_auth_mode(ext_name.clone()); thread.complete_turn(&instructions); } } @@ -1391,7 +1762,12 @@ impl Agent { .channels .send_status( &message.channel, - StatusUpdate::Status("Awaiting token".into()), + StatusUpdate::AuthRequired { + extension_name: ext_name, + instructions: Some(instructions.clone()), + auth_url: auth_data.auth_url, + setup_url: auth_data.setup_url, + }, &message.metadata, ) .await; @@ -1434,6 +1810,7 @@ impl Agent { match result { Ok(AgenticLoopResult::Response(response)) => { thread.complete_turn(&response); + self.persist_response_chain(thread); let _ = self .channels .send_status( @@ -1532,16 +1909,6 @@ impl Agent { pending.extension_name ); - // Notify via channel status so the response doesn't echo the token - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::Status("Authenticated, loading tools...".into()), - &message.metadata, - ) - .await; - // Auto-activate so tools are available immediately after auth match ext_mgr.activate(&pending.extension_name).await { Ok(activate_result) => { @@ -1551,10 +1918,23 @@ impl Agent { } else { format!("\n\nTools: {}", activate_result.tools_loaded.join(", ")) }; - Ok(Some(format!( + let msg = format!( "{} authenticated and activated ({} tools loaded).{}", pending.extension_name, tool_count, tool_list - ))) + ); + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthCompleted { + extension_name: pending.extension_name.clone(), + success: true, + message: msg.clone(), + }, + &message.metadata, + ) + .await; + Ok(Some(msg)) } Err(e) => { tracing::warn!( @@ -1562,16 +1942,29 @@ impl Agent { pending.extension_name, e ); - Ok(Some(format!( + let msg = format!( "{} authenticated successfully, but activation failed: {}. \ Try activating manually.", pending.extension_name, e - ))) + ); + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthCompleted { + extension_name: pending.extension_name.clone(), + success: true, + message: msg.clone(), + }, + &message.metadata, + ) + .await; + Ok(Some(msg)) } } } Ok(result) => { - // Unexpected state, re-enter auth mode + // Invalid token, re-enter auth mode { let mut sess = session.lock().await; if let Some(thread) = sess.threads.get_mut(&thread_id) { @@ -1580,13 +1973,43 @@ impl Agent { } let msg = result .instructions + .clone() .unwrap_or_else(|| "Invalid token. Please try again.".to_string()); + // Re-emit AuthRequired so web UI re-shows the card + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthRequired { + extension_name: pending.extension_name.clone(), + instructions: Some(msg.clone()), + auth_url: result.auth_url, + setup_url: result.setup_url, + }, + &message.metadata, + ) + .await; + Ok(Some(msg)) + } + Err(e) => { + let msg = format!( + "Authentication failed for {}: {}", + pending.extension_name, e + ); + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthCompleted { + extension_name: pending.extension_name.clone(), + success: false, + message: msg.clone(), + }, + &message.metadata, + ) + .await; Ok(Some(msg)) } - Err(e) => Ok(Some(format!( - "Authentication failed for {}: {}", - pending.extension_name, e - ))), } } @@ -1676,15 +2099,15 @@ impl Agent { } // Persist new job to database (fire-and-forget) - if let Some(store) = self.store() { - if let Ok(ctx) = self.context_manager.get_context(job_id).await { - let store = store.clone(); - tokio::spawn(async move { - if let Err(e) = store.save_job(&ctx).await { - tracing::warn!("Failed to persist new job {}: {}", job_id, e); - } - }); - } + if let Some(store) = self.store() + && let Ok(ctx) = self.context_manager.get_context(job_id).await + { + let store = store.clone(); + tokio::spawn(async move { + if let Err(e) = store.save_job(&ctx).await { + tracing::warn!("Failed to persist new job {}: {}", job_id, e); + } + }); } // Schedule for execution @@ -1764,10 +2187,10 @@ impl Agent { let mut output = String::from("Jobs:\n"); for job_id in jobs { - if let Ok(ctx) = self.context_manager.get_context(job_id).await { - if ctx.user_id == user_id { - output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state)); - } + if let Ok(ctx) = self.context_manager.get_context(job_id).await + && ctx.user_id == user_id + { + output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state)); } } @@ -1937,40 +2360,49 @@ impl Agent { } } - async fn handle_command( + /// Handle system commands that bypass thread-state checks entirely. + async fn handle_system_command( &self, command: &str, - _args: &[String], - ) -> Result, Error> { + args: &[String], + ) -> Result { match command { - "help" => Ok(Some( - r#"Commands: - /job - Create a job - /status [id] - Check job status - /cancel - Cancel a job - /list - List all jobs - /help - Help a stuck job + "help" => Ok(SubmissionResult::response(concat!( + "System:\n", + " /help Show this help\n", + " /model [name] Show or switch the active model\n", + " /version Show version info\n", + " /tools List available tools\n", + " /debug Toggle debug mode\n", + " /ping Connectivity check\n", + "\n", + "Jobs:\n", + " /job Create a new job\n", + " /status [id] Check job status\n", + " /cancel Cancel a job\n", + " /list List all jobs\n", + "\n", + "Session:\n", + " /undo Undo last turn\n", + " /redo Redo undone turn\n", + " /compact Compress context window\n", + " /clear Clear current thread\n", + " /interrupt Stop current operation\n", + " /new New conversation thread\n", + " /thread Switch to thread\n", + " /resume Resume from checkpoint\n", + "\n", + "Agent:\n", + " /heartbeat Run heartbeat check\n", + " /summarize Summarize current thread\n", + " /suggest Suggest next steps\n", + "\n", + " /quit Exit", + ))), - /undo - Undo last turn - /redo - Redo undone turn - /compact - Compress context - /clear - Clear thread - /interrupt - Stop current turn - /thread new - New thread - /thread - Switch thread - /resume - Resume checkpoint + "ping" => Ok(SubmissionResult::response("pong!")), - /heartbeat - Run heartbeat check now - /summarize - Summarize current thread - /suggest - Suggest next steps - - /quit - Exit"# - .to_string(), - )), - - "ping" => Ok(Some("pong!".to_string())), - - "version" => Ok(Some(format!( + "version" => Ok(SubmissionResult::response(format!( "{} v{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION") @@ -1978,12 +2410,113 @@ impl Agent { "tools" => { let tools = self.tools().list().await; - Ok(Some(format!("Available tools: {}", tools.join(", ")))) + Ok(SubmissionResult::response(format!( + "Available tools: {}", + tools.join(", ") + ))) } - _ => Ok(Some(format!("Unknown command: {}. Try /help", command))), + "debug" => { + // Debug toggle is handled client-side in the REPL. + // For non-REPL channels, just acknowledge. + Ok(SubmissionResult::ok_with_message( + "Debug toggle is handled by your client.", + )) + } + + "model" => { + if args.is_empty() { + // Show current model + let name = self.llm().active_model_name(); + Ok(SubmissionResult::response(format!( + "Active model: {}", + name + ))) + } else { + let requested = &args[0]; + + // Validate the model exists + match self.llm().list_models().await { + Ok(models) if !models.is_empty() => { + if !models.iter().any(|m| m == requested) { + return Ok(SubmissionResult::error(format!( + "Unknown model: {}. Available models:\n {}", + requested, + models.join("\n ") + ))); + } + } + Ok(_) => { + // Empty model list, can't validate but try anyway + } + Err(e) => { + tracing::warn!("Could not fetch model list for validation: {}", e); + // Proceed anyway, the provider will error on the next call if invalid + } + } + + match self.llm().set_model(requested) { + Ok(()) => Ok(SubmissionResult::response(format!( + "Switched model to: {}", + requested + ))), + Err(e) => Ok(SubmissionResult::error(format!( + "Failed to switch model: {}", + e + ))), + } + } + } + + _ => Ok(SubmissionResult::error(format!( + "Unknown command: {}. Try /help", + command + ))), } } + + /// Handle legacy command routing from the Router (job commands that go through + /// process_user_input -> router -> handle_job_or_command -> here). + async fn handle_command( + &self, + command: &str, + args: &[String], + ) -> Result, Error> { + // System commands are now handled directly via Submission::SystemCommand, + // but the router may still send us unknown /commands. + match self.handle_system_command(command, args).await? { + SubmissionResult::Response { content } => Ok(Some(content)), + SubmissionResult::Ok { message } => Ok(message), + SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))), + _ => Ok(None), + } + } +} + +/// Parsed auth result fields for emitting StatusUpdate::AuthRequired. +struct ParsedAuthData { + auth_url: Option, + setup_url: Option, +} + +/// Extract auth_url and setup_url from a tool_auth result JSON string. +fn parse_auth_result(result: &Result) -> ParsedAuthData { + let parsed = result + .as_ref() + .ok() + .and_then(|s| serde_json::from_str::(s).ok()); + ParsedAuthData { + auth_url: parsed + .as_ref() + .and_then(|v| v.get("auth_url")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + setup_url: parsed + .as_ref() + .and_then(|v| v.get("setup_url")) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + } } /// Check if a tool_auth result indicates the extension is awaiting a token. @@ -1994,7 +2527,7 @@ fn detect_auth_awaiting( tool_name: &str, result: &Result, ) -> Option<(String, String)> { - if tool_name != "tool_auth" { + if tool_name != "tool_auth" && tool_name != "tool_activate" { return None; } let output = result.as_ref().ok()?; @@ -2078,4 +2611,100 @@ mod tests { let (_, instructions) = detect_auth_awaiting("tool_auth", &result).unwrap(); assert_eq!(instructions, "Please provide your API token/key."); } + + #[test] + fn test_detect_auth_awaiting_tool_activate() { + let result: Result = Ok(serde_json::json!({ + "name": "slack", + "kind": "McpServer", + "awaiting_token": true, + "status": "awaiting_token", + "instructions": "Provide your Slack Bot token." + }) + .to_string()); + + let detected = detect_auth_awaiting("tool_activate", &result); + assert!(detected.is_some()); + let (name, instructions) = detected.unwrap(); + assert_eq!(name, "slack"); + assert!(instructions.contains("Slack Bot")); + } + + #[test] + fn test_detect_auth_awaiting_tool_activate_not_awaiting() { + let result: Result = Ok(serde_json::json!({ + "name": "slack", + "tools_loaded": ["slack_post_message"], + "message": "Activated" + }) + .to_string()); + + assert!(detect_auth_awaiting("tool_activate", &result).is_none()); + } + + // --- truncate_for_preview tests --- + + use super::truncate_for_preview; + + #[test] + fn test_truncate_short_input() { + assert_eq!(truncate_for_preview("hello", 10), "hello"); + } + + #[test] + fn test_truncate_empty_input() { + assert_eq!(truncate_for_preview("", 10), ""); + } + + #[test] + fn test_truncate_exact_length() { + assert_eq!(truncate_for_preview("hello", 5), "hello"); + } + + #[test] + fn test_truncate_over_limit() { + let result = truncate_for_preview("hello world, this is long", 10); + assert!(result.ends_with("...")); + // "hello worl" = 10 chars + "..." + assert_eq!(result, "hello worl..."); + } + + #[test] + fn test_truncate_collapses_newlines() { + let result = truncate_for_preview("line1\nline2\nline3", 100); + assert!(!result.contains('\n')); + assert_eq!(result, "line1 line2 line3"); + } + + #[test] + fn test_truncate_collapses_whitespace() { + let result = truncate_for_preview("hello world", 100); + assert_eq!(result, "hello world"); + } + + #[test] + fn test_truncate_multibyte_utf8() { + // Each emoji is 4 bytes. Truncating at char boundary must not panic. + let input = "😀😁😂🤣😃😄😅😆😉😊"; + let result = truncate_for_preview(input, 5); + assert!(result.ends_with("...")); + // First 5 chars = 5 emoji + assert_eq!(result, "😀😁😂🤣😃..."); + } + + #[test] + fn test_truncate_cjk_characters() { + // CJK chars are 3 bytes each in UTF-8. + let input = "你好世界测试数据很长的字符串"; + let result = truncate_for_preview(input, 4); + assert_eq!(result, "你好世界..."); + } + + #[test] + fn test_truncate_mixed_multibyte_and_ascii() { + let input = "hello 世界 foo"; + let result = truncate_for_preview(input, 8); + // 'h','e','l','l','o',' ','世','界' = 8 chars + assert_eq!(result, "hello 世界..."); + } } diff --git a/src/agent/heartbeat.rs b/src/agent/heartbeat.rs index 115b8159..ff35955d 100644 --- a/src/agent/heartbeat.rs +++ b/src/agent/heartbeat.rs @@ -29,7 +29,7 @@ use std::time::Duration; use tokio::sync::mpsc; use crate::channels::OutgoingResponse; -use crate::llm::{ChatMessage, CompletionRequest, LlmProvider}; +use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; use crate::workspace::Workspace; /// Configuration for the heartbeat runner. @@ -217,9 +217,26 @@ impl HeartbeatRunner { ] }; + // Use the model's context_length to set max_tokens. The API returns + // the total context window; we cap output at half of that (the rest is + // the prompt) with a floor of 4096. + let max_tokens = match self.llm.model_metadata().await { + Ok(meta) => { + let from_api = meta.context_length.map(|ctx| ctx / 2).unwrap_or(4096); + from_api.max(4096) + } + Err(e) => { + tracing::warn!( + "Could not fetch model metadata, using default max_tokens: {}", + e + ); + 4096 + } + }; + let request = CompletionRequest::new(messages) - .with_max_tokens(1024) - .with_temperature(0.3); // Lower temperature for more focused responses + .with_max_tokens(max_tokens) + .with_temperature(0.3); let response = match self.llm.complete(request).await { Ok(r) => r, @@ -228,6 +245,20 @@ impl HeartbeatRunner { let content = response.content.trim(); + // Guard against empty content. Reasoning models (e.g. GLM-4.7) may + // burn all output tokens on chain-of-thought and return content: null. + if content.is_empty() { + return if response.finish_reason == FinishReason::Length { + HeartbeatResult::Failed( + "LLM response was truncated (finish_reason=length) with no content. \ + The model may have exhausted its token budget on reasoning." + .to_string(), + ) + } else { + HeartbeatResult::Failed("LLM returned empty content.".to_string()) + }; + } + // Check if nothing needs attention if content == "HEARTBEAT_OK" || content.contains("HEARTBEAT_OK") { return HeartbeatResult::Ok; diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 7c3d7685..a667b17d 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -6,6 +6,7 @@ //! - Tool invocation with safety //! - Self-repair for stuck jobs //! - Proactive heartbeat execution +//! - Routine-based scheduled and reactive jobs //! - Turn-based session management with undo //! - Context compaction for long conversations @@ -14,6 +15,8 @@ pub mod compaction; pub mod context_monitor; mod heartbeat; mod router; +pub mod routine; +pub mod routine_engine; mod scheduler; mod self_repair; pub mod session; @@ -23,11 +26,14 @@ pub mod task; pub mod undo; pub mod worker; +pub(crate) use agent_loop::truncate_for_preview; pub use agent_loop::{Agent, AgentDeps}; pub use compaction::{CompactionResult, ContextCompactor}; pub use context_monitor::{CompactionStrategy, ContextBreakdown, ContextMonitor}; pub use heartbeat::{HeartbeatConfig, HeartbeatResult, HeartbeatRunner, spawn_heartbeat}; pub use router::{MessageIntent, Router}; +pub use routine::{Routine, RoutineAction, RoutineRun, Trigger}; +pub use routine_engine::RoutineEngine; pub use scheduler::Scheduler; pub use self_repair::{BrokenTool, RepairResult, RepairTask, SelfRepair, StuckJob}; pub use session::{PendingApproval, PendingAuth, Session, Thread, ThreadState, Turn, TurnState}; diff --git a/src/agent/routine.rs b/src/agent/routine.rs new file mode 100644 index 00000000..084a9b9f --- /dev/null +++ b/src/agent/routine.rs @@ -0,0 +1,509 @@ +//! Core types for the routines system. +//! +//! A routine is a named, persistent, user-owned task with a trigger and an action. +//! Each routine fires independently when its trigger condition is met, with only +//! that routine's prompt and context sent to the LLM. +//! +//! ```text +//! ┌──────────┐ ┌─────────┐ ┌──────────────────┐ +//! │ Trigger │────▶│ Engine │────▶│ Execution Mode │ +//! │ cron/event│ │guardrail│ │lightweight│full_job│ +//! │ webhook │ │ check │ └──────────────────┘ +//! │ manual │ └─────────┘ │ +//! └──────────┘ ▼ +//! ┌──────────────┐ +//! │ Notify user │ +//! │ if needed │ +//! └──────────────┘ +//! ``` + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::str::FromStr; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +/// A routine is a named, persistent, user-owned task with a trigger and an action. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Routine { + pub id: Uuid, + pub name: String, + pub description: String, + pub user_id: String, + pub enabled: bool, + pub trigger: Trigger, + pub action: RoutineAction, + pub guardrails: RoutineGuardrails, + pub notify: NotifyConfig, + + // Runtime state (DB-managed) + pub last_run_at: Option>, + pub next_fire_at: Option>, + pub run_count: u64, + pub consecutive_failures: u32, + pub state: serde_json::Value, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// When a routine should fire. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Trigger { + /// Fire on a cron schedule (e.g. "0 9 * * MON-FRI" or "every 2h"). + Cron { schedule: String }, + /// Fire when a channel message matches a pattern. + Event { + /// Optional channel filter (e.g. "telegram", "slack"). + channel: Option, + /// Regex pattern to match against message content. + pattern: String, + }, + /// Fire on incoming webhook POST to /hooks/routine/{id}. + Webhook { + /// Optional webhook path suffix (defaults to routine id). + path: Option, + /// Optional shared secret for HMAC validation. + secret: Option, + }, + /// Only fires via tool call or CLI. + Manual, +} + +impl Trigger { + /// The string tag stored in the DB trigger_type column. + pub fn type_tag(&self) -> &'static str { + match self { + Trigger::Cron { .. } => "cron", + Trigger::Event { .. } => "event", + Trigger::Webhook { .. } => "webhook", + Trigger::Manual => "manual", + } + } + + /// Parse a trigger from its DB representation. + pub fn from_db(trigger_type: &str, config: serde_json::Value) -> Result { + match trigger_type { + "cron" => { + let schedule = config + .get("schedule") + .and_then(|v| v.as_str()) + .ok_or("cron trigger missing 'schedule'")? + .to_string(); + Ok(Trigger::Cron { schedule }) + } + "event" => { + let pattern = config + .get("pattern") + .and_then(|v| v.as_str()) + .ok_or("event trigger missing 'pattern'")? + .to_string(); + let channel = config + .get("channel") + .and_then(|v| v.as_str()) + .map(String::from); + Ok(Trigger::Event { channel, pattern }) + } + "webhook" => { + let path = config + .get("path") + .and_then(|v| v.as_str()) + .map(String::from); + let secret = config + .get("secret") + .and_then(|v| v.as_str()) + .map(String::from); + Ok(Trigger::Webhook { path, secret }) + } + "manual" => Ok(Trigger::Manual), + other => Err(format!("unknown trigger type: {other}")), + } + } + + /// Serialize trigger-specific config to JSON for DB storage. + pub fn to_config_json(&self) -> serde_json::Value { + match self { + Trigger::Cron { schedule } => serde_json::json!({ "schedule": schedule }), + Trigger::Event { channel, pattern } => serde_json::json!({ + "pattern": pattern, + "channel": channel, + }), + Trigger::Webhook { path, secret } => serde_json::json!({ + "path": path, + "secret": secret, + }), + Trigger::Manual => serde_json::json!({}), + } + } +} + +/// What happens when a routine fires. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RoutineAction { + /// Single LLM call, no tools. Cheap and fast. + Lightweight { + /// The prompt sent to the LLM. + prompt: String, + /// Workspace paths to load as context (e.g. ["context/priorities.md"]). + #[serde(default)] + context_paths: Vec, + /// Max output tokens (default: 4096). + #[serde(default = "default_max_tokens")] + max_tokens: u32, + }, + /// Full multi-turn worker job with tool access. + FullJob { + /// Job title for the scheduler. + title: String, + /// Job description / initial prompt. + description: String, + /// Max reasoning iterations (default: 10). + #[serde(default = "default_max_iterations")] + max_iterations: u32, + }, +} + +fn default_max_tokens() -> u32 { + 4096 +} + +fn default_max_iterations() -> u32 { + 10 +} + +impl RoutineAction { + /// The string tag stored in the DB action_type column. + pub fn type_tag(&self) -> &'static str { + match self { + RoutineAction::Lightweight { .. } => "lightweight", + RoutineAction::FullJob { .. } => "full_job", + } + } + + /// Parse an action from its DB representation. + pub fn from_db(action_type: &str, config: serde_json::Value) -> Result { + match action_type { + "lightweight" => { + let prompt = config + .get("prompt") + .and_then(|v| v.as_str()) + .ok_or("lightweight action missing 'prompt'")? + .to_string(); + let context_paths = config + .get("context_paths") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default(); + let max_tokens = config + .get("max_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(default_max_tokens() as u64) as u32; + Ok(RoutineAction::Lightweight { + prompt, + context_paths, + max_tokens, + }) + } + "full_job" => { + let title = config + .get("title") + .and_then(|v| v.as_str()) + .ok_or("full_job action missing 'title'")? + .to_string(); + let description = config + .get("description") + .and_then(|v| v.as_str()) + .ok_or("full_job action missing 'description'")? + .to_string(); + let max_iterations = config + .get("max_iterations") + .and_then(|v| v.as_u64()) + .unwrap_or(default_max_iterations() as u64) + as u32; + Ok(RoutineAction::FullJob { + title, + description, + max_iterations, + }) + } + other => Err(format!("unknown action type: {other}")), + } + } + + /// Serialize action config to JSON for DB storage. + pub fn to_config_json(&self) -> serde_json::Value { + match self { + RoutineAction::Lightweight { + prompt, + context_paths, + max_tokens, + } => serde_json::json!({ + "prompt": prompt, + "context_paths": context_paths, + "max_tokens": max_tokens, + }), + RoutineAction::FullJob { + title, + description, + max_iterations, + } => serde_json::json!({ + "title": title, + "description": description, + "max_iterations": max_iterations, + }), + } + } +} + +/// Guardrails to prevent runaway execution. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoutineGuardrails { + /// Minimum time between fires. + pub cooldown: Duration, + /// Max simultaneous runs of this routine. + pub max_concurrent: u32, + /// Window for content-hash dedup (event triggers). None = no dedup. + pub dedup_window: Option, +} + +impl Default for RoutineGuardrails { + fn default() -> Self { + Self { + cooldown: Duration::from_secs(300), + max_concurrent: 1, + dedup_window: None, + } + } +} + +/// Notification preferences for a routine. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NotifyConfig { + /// Channel to notify on (None = default/broadcast all). + pub channel: Option, + /// User to notify. + pub user: String, + /// Notify when routine produces actionable output. + pub on_attention: bool, + /// Notify when routine errors. + pub on_failure: bool, + /// Notify when routine runs with no findings. + pub on_success: bool, +} + +impl Default for NotifyConfig { + fn default() -> Self { + Self { + channel: None, + user: "default".to_string(), + on_attention: true, + on_failure: true, + on_success: false, + } + } +} + +/// Status of a routine run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RunStatus { + Running, + Ok, + Attention, + Failed, +} + +impl std::fmt::Display for RunStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RunStatus::Running => write!(f, "running"), + RunStatus::Ok => write!(f, "ok"), + RunStatus::Attention => write!(f, "attention"), + RunStatus::Failed => write!(f, "failed"), + } + } +} + +impl FromStr for RunStatus { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "running" => Ok(RunStatus::Running), + "ok" => Ok(RunStatus::Ok), + "attention" => Ok(RunStatus::Attention), + "failed" => Ok(RunStatus::Failed), + other => Err(format!("unknown run status: {other}")), + } + } +} + +/// A single execution of a routine. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoutineRun { + pub id: Uuid, + pub routine_id: Uuid, + pub trigger_type: String, + pub trigger_detail: Option, + pub started_at: DateTime, + pub completed_at: Option>, + pub status: RunStatus, + pub result_summary: Option, + pub tokens_used: Option, + pub job_id: Option, + pub created_at: DateTime, +} + +/// Compute a content hash for event dedup. +pub fn content_hash(content: &str) -> u64 { + let mut hasher = DefaultHasher::new(); + content.hash(&mut hasher); + hasher.finish() +} + +/// Parse a cron expression and compute the next fire time from now. +pub fn next_cron_fire(schedule: &str) -> Result>, String> { + let cron_schedule = + cron::Schedule::from_str(schedule).map_err(|e| format!("invalid cron: {e}"))?; + Ok(cron_schedule.upcoming(Utc).next()) +} + +#[cfg(test)] +mod tests { + use crate::agent::routine::{ + RoutineAction, RoutineGuardrails, RunStatus, Trigger, content_hash, next_cron_fire, + }; + + #[test] + fn test_trigger_roundtrip() { + let trigger = Trigger::Cron { + schedule: "0 9 * * MON-FRI".to_string(), + }; + let json = trigger.to_config_json(); + let parsed = Trigger::from_db("cron", json).expect("parse cron"); + assert!(matches!(parsed, Trigger::Cron { schedule } if schedule == "0 9 * * MON-FRI")); + } + + #[test] + fn test_event_trigger_roundtrip() { + let trigger = Trigger::Event { + channel: Some("telegram".to_string()), + pattern: r"deploy\s+\w+".to_string(), + }; + let json = trigger.to_config_json(); + let parsed = Trigger::from_db("event", json).expect("parse event"); + assert!(matches!(parsed, Trigger::Event { channel, pattern } + if channel == Some("telegram".to_string()) && pattern == r"deploy\s+\w+")); + } + + #[test] + fn test_action_lightweight_roundtrip() { + let action = RoutineAction::Lightweight { + prompt: "Check PRs".to_string(), + context_paths: vec!["context/priorities.md".to_string()], + max_tokens: 2048, + }; + let json = action.to_config_json(); + let parsed = RoutineAction::from_db("lightweight", json).expect("parse lightweight"); + assert!( + matches!(parsed, RoutineAction::Lightweight { prompt, context_paths, max_tokens } + if prompt == "Check PRs" && context_paths.len() == 1 && max_tokens == 2048) + ); + } + + #[test] + fn test_action_full_job_roundtrip() { + let action = RoutineAction::FullJob { + title: "Deploy review".to_string(), + description: "Review and deploy pending changes".to_string(), + max_iterations: 5, + }; + let json = action.to_config_json(); + let parsed = RoutineAction::from_db("full_job", json).expect("parse full_job"); + assert!( + matches!(parsed, RoutineAction::FullJob { title, max_iterations, .. } + if title == "Deploy review" && max_iterations == 5) + ); + } + + #[test] + fn test_run_status_display_parse() { + for status in [ + RunStatus::Running, + RunStatus::Ok, + RunStatus::Attention, + RunStatus::Failed, + ] { + let s = status.to_string(); + let parsed: RunStatus = s.parse().expect("parse status"); + assert_eq!(parsed, status); + } + } + + #[test] + fn test_content_hash_deterministic() { + let h1 = content_hash("deploy production"); + let h2 = content_hash("deploy production"); + assert_eq!(h1, h2); + + let h3 = content_hash("deploy staging"); + assert_ne!(h1, h3); + } + + #[test] + fn test_next_cron_fire_valid() { + // Every minute should always have a next fire + let next = next_cron_fire("* * * * * *").expect("valid cron"); + assert!(next.is_some()); + } + + #[test] + fn test_next_cron_fire_invalid() { + let result = next_cron_fire("not a cron"); + assert!(result.is_err()); + } + + #[test] + fn test_guardrails_default() { + let g = RoutineGuardrails::default(); + assert_eq!(g.cooldown.as_secs(), 300); + assert_eq!(g.max_concurrent, 1); + assert!(g.dedup_window.is_none()); + } + + #[test] + fn test_trigger_type_tag() { + assert_eq!( + Trigger::Cron { + schedule: String::new() + } + .type_tag(), + "cron" + ); + assert_eq!( + Trigger::Event { + channel: None, + pattern: String::new() + } + .type_tag(), + "event" + ); + assert_eq!( + Trigger::Webhook { + path: None, + secret: None + } + .type_tag(), + "webhook" + ); + assert_eq!(Trigger::Manual.type_tag(), "manual"); + } +} diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs new file mode 100644 index 00000000..52156ac5 --- /dev/null +++ b/src/agent/routine_engine.rs @@ -0,0 +1,601 @@ +//! Routine execution engine. +//! +//! Handles loading routines, checking triggers, enforcing guardrails, +//! and executing both lightweight (single LLM call) and full-job routines. +//! +//! The engine runs two independent loops: +//! - A **cron ticker** that polls the DB every N seconds for due cron routines +//! - An **event matcher** called synchronously from the agent main loop +//! +//! Lightweight routines execute inline (single LLM call, no scheduler slot). +//! Full-job routines are delegated to the existing `Scheduler`. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use chrono::Utc; +use regex::Regex; +use tokio::sync::{RwLock, mpsc}; +use uuid::Uuid; + +use crate::agent::routine::{ + NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, next_cron_fire, +}; +use crate::channels::{IncomingMessage, OutgoingResponse}; +use crate::config::RoutineConfig; +use crate::db::Database; +use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; +use crate::workspace::Workspace; + +/// The routine execution engine. +pub struct RoutineEngine { + config: RoutineConfig, + store: Arc, + llm: Arc, + workspace: Arc, + /// Sender for notifications (routed to channel manager). + notify_tx: mpsc::Sender, + /// Currently running routine count (across all routines). + running_count: Arc, + /// Compiled event regex cache: routine_id -> compiled regex. + event_cache: Arc>>, +} + +impl RoutineEngine { + pub fn new( + config: RoutineConfig, + store: Arc, + llm: Arc, + workspace: Arc, + notify_tx: mpsc::Sender, + ) -> Self { + Self { + config, + store, + llm, + workspace, + notify_tx, + running_count: Arc::new(AtomicUsize::new(0)), + event_cache: Arc::new(RwLock::new(Vec::new())), + } + } + + /// Refresh the in-memory event trigger cache from DB. + pub async fn refresh_event_cache(&self) { + match self.store.list_event_routines().await { + Ok(routines) => { + let mut cache = Vec::new(); + for routine in routines { + if let Trigger::Event { ref pattern, .. } = routine.trigger { + match Regex::new(pattern) { + Ok(re) => cache.push((routine.id, routine.clone(), re)), + Err(e) => { + tracing::warn!( + routine = %routine.name, + "Invalid event regex '{}': {}", + pattern, e + ); + } + } + } + } + let count = cache.len(); + *self.event_cache.write().await = cache; + tracing::debug!("Refreshed event cache: {} routines", count); + } + Err(e) => { + tracing::error!("Failed to refresh event cache: {}", e); + } + } + } + + /// Check incoming message against event triggers. Returns number of routines fired. + /// + /// Called synchronously from the main loop after handle_message(). The actual + /// execution is spawned async so this returns quickly. + pub async fn check_event_triggers(&self, message: &IncomingMessage) -> usize { + let cache = self.event_cache.read().await; + let mut fired = 0; + + for (_, routine, re) in cache.iter() { + // Channel filter + if let Trigger::Event { + channel: Some(ch), .. + } = &routine.trigger + && ch != &message.channel + { + continue; + } + + // Regex match + if !re.is_match(&message.content) { + continue; + } + + // Cooldown check + if !self.check_cooldown(routine) { + tracing::debug!(routine = %routine.name, "Skipped: cooldown active"); + continue; + } + + // Concurrent run check + if !self.check_concurrent(routine).await { + tracing::debug!(routine = %routine.name, "Skipped: max concurrent reached"); + continue; + } + + // Global capacity check + if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines { + tracing::warn!(routine = %routine.name, "Skipped: global max concurrent reached"); + continue; + } + + let detail = truncate(&message.content, 200); + self.spawn_fire(routine.clone(), "event", Some(detail)); + fired += 1; + } + + fired + } + + /// Check all due cron routines and fire them. Called by the cron ticker. + pub async fn check_cron_triggers(&self) { + let routines = match self.store.list_due_cron_routines().await { + Ok(r) => r, + Err(e) => { + tracing::error!("Failed to load due cron routines: {}", e); + return; + } + }; + + for routine in routines { + if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines { + tracing::warn!("Global max concurrent routines reached, skipping remaining"); + break; + } + + if !self.check_cooldown(&routine) { + continue; + } + + if !self.check_concurrent(&routine).await { + continue; + } + + let detail = if let Trigger::Cron { ref schedule } = routine.trigger { + Some(schedule.clone()) + } else { + None + }; + + self.spawn_fire(routine, "cron", detail); + } + } + + /// Fire a routine manually (from tool call or CLI). + pub async fn fire_manual(&self, routine_id: Uuid) -> Result { + let routine = self + .store + .get_routine(routine_id) + .await + .map_err(|e| format!("DB error: {e}"))? + .ok_or_else(|| format!("routine {routine_id} not found"))?; + + if !routine.enabled { + return Err(format!("routine '{}' is disabled", routine.name)); + } + + if !self.check_concurrent(&routine).await { + return Err(format!( + "routine '{}' already at max concurrent runs", + routine.name + )); + } + + let run_id = Uuid::new_v4(); + let run = RoutineRun { + id: run_id, + routine_id: routine.id, + trigger_type: "manual".to_string(), + trigger_detail: None, + started_at: Utc::now(), + completed_at: None, + status: RunStatus::Running, + result_summary: None, + tokens_used: None, + job_id: None, + created_at: Utc::now(), + }; + + if let Err(e) = self.store.create_routine_run(&run).await { + return Err(format!("failed to create run record: {e}")); + } + + // Execute inline for manual triggers (caller wants to wait) + let engine = EngineContext { + store: self.store.clone(), + llm: self.llm.clone(), + workspace: self.workspace.clone(), + notify_tx: self.notify_tx.clone(), + running_count: self.running_count.clone(), + max_lightweight_tokens: self.config.max_lightweight_tokens, + }; + + tokio::spawn(async move { + execute_routine(engine, routine, run).await; + }); + + Ok(run_id) + } + + /// Spawn a fire in a background task. + fn spawn_fire(&self, routine: Routine, trigger_type: &str, trigger_detail: Option) { + let run = RoutineRun { + id: Uuid::new_v4(), + routine_id: routine.id, + trigger_type: trigger_type.to_string(), + trigger_detail, + started_at: Utc::now(), + completed_at: None, + status: RunStatus::Running, + result_summary: None, + tokens_used: None, + job_id: None, + created_at: Utc::now(), + }; + + let engine = EngineContext { + store: self.store.clone(), + llm: self.llm.clone(), + workspace: self.workspace.clone(), + notify_tx: self.notify_tx.clone(), + running_count: self.running_count.clone(), + max_lightweight_tokens: self.config.max_lightweight_tokens, + }; + + // Record the run in DB, then spawn execution + let store = self.store.clone(); + tokio::spawn(async move { + if let Err(e) = store.create_routine_run(&run).await { + tracing::error!(routine = %routine.name, "Failed to record run: {}", e); + return; + } + execute_routine(engine, routine, run).await; + }); + } + + fn check_cooldown(&self, routine: &Routine) -> bool { + if let Some(last_run) = routine.last_run_at { + let elapsed = Utc::now().signed_duration_since(last_run); + let cooldown = chrono::Duration::from_std(routine.guardrails.cooldown) + .unwrap_or(chrono::Duration::seconds(300)); + if elapsed < cooldown { + return false; + } + } + true + } + + async fn check_concurrent(&self, routine: &Routine) -> bool { + match self.store.count_running_routine_runs(routine.id).await { + Ok(count) => count < routine.guardrails.max_concurrent as i64, + Err(e) => { + tracing::error!( + routine = %routine.name, + "Failed to check concurrent runs: {}", e + ); + false + } + } + } +} + +/// Shared context passed to the execution function. +struct EngineContext { + store: Arc, + llm: Arc, + workspace: Arc, + notify_tx: mpsc::Sender, + running_count: Arc, + max_lightweight_tokens: u32, +} + +/// Execute a routine run. Handles both lightweight and full_job modes. +async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) { + // Increment running count (atomic: survives panics in the execution below) + ctx.running_count.fetch_add(1, Ordering::Relaxed); + + let result = match &routine.action { + RoutineAction::Lightweight { + prompt, + context_paths, + max_tokens, + } => execute_lightweight(&ctx, &routine, prompt, context_paths, *max_tokens).await, + RoutineAction::FullJob { description, .. } => { + // Full job mode: for now, execute as lightweight with the description + // as prompt. Full scheduler integration will come as a follow-up. + tracing::info!( + routine = %routine.name, + "FullJob mode executing as lightweight (scheduler integration pending)" + ); + execute_lightweight(&ctx, &routine, description, &[], ctx.max_lightweight_tokens).await + } + }; + + // Decrement running count + ctx.running_count.fetch_sub(1, Ordering::Relaxed); + + // Process result + let (status, summary, tokens) = match result { + Ok(execution) => execution, + Err(e) => { + tracing::error!(routine = %routine.name, "Execution failed: {}", e); + (RunStatus::Failed, Some(e), None) + } + }; + + // Complete the run record + if let Err(e) = ctx + .store + .complete_routine_run(run.id, status, summary.as_deref(), tokens) + .await + { + tracing::error!(routine = %routine.name, "Failed to complete run record: {}", e); + } + + // Update routine runtime state + let now = Utc::now(); + let next_fire = if let Trigger::Cron { ref schedule } = routine.trigger { + next_cron_fire(schedule).unwrap_or(None) + } else { + None + }; + + let new_failures = if status == RunStatus::Failed { + routine.consecutive_failures + 1 + } else { + 0 + }; + + if let Err(e) = ctx + .store + .update_routine_runtime( + routine.id, + now, + next_fire, + routine.run_count + 1, + new_failures, + &routine.state, + ) + .await + { + tracing::error!(routine = %routine.name, "Failed to update runtime state: {}", e); + } + + // Send notifications based on config + send_notification( + &ctx.notify_tx, + &routine.notify, + &routine.name, + status, + summary.as_deref(), + ) + .await; +} + +/// Execute a lightweight routine (single LLM call). +async fn execute_lightweight( + ctx: &EngineContext, + routine: &Routine, + prompt: &str, + context_paths: &[String], + max_tokens: u32, +) -> Result<(RunStatus, Option, Option), String> { + // Load context from workspace + let mut context_parts = Vec::new(); + for path in context_paths { + match ctx.workspace.read(path).await { + Ok(doc) => { + context_parts.push(format!("## {}\n\n{}", path, doc.content)); + } + Err(e) => { + tracing::debug!( + routine = %routine.name, + "Failed to read context path {}: {}", path, e + ); + } + } + } + + // Load routine state from workspace + let state_path = format!("routines/{}/state.md", routine.name); + let state_content = match ctx.workspace.read(&state_path).await { + Ok(doc) => Some(doc.content), + Err(_) => None, + }; + + // Build the prompt + let mut full_prompt = String::new(); + full_prompt.push_str(prompt); + + if !context_parts.is_empty() { + full_prompt.push_str("\n\n---\n\n# Context\n\n"); + full_prompt.push_str(&context_parts.join("\n\n")); + } + + if let Some(state) = &state_content { + full_prompt.push_str("\n\n---\n\n# Previous State\n\n"); + full_prompt.push_str(state); + } + + full_prompt.push_str( + "\n\n---\n\nIf nothing needs attention, reply EXACTLY with: ROUTINE_OK\n\ + If something needs attention, provide a concise summary.", + ); + + // Get system prompt + let system_prompt = match ctx.workspace.system_prompt().await { + Ok(p) => p, + Err(e) => { + tracing::warn!(routine = %routine.name, "Failed to get system prompt: {}", e); + String::new() + } + }; + + let messages = if system_prompt.is_empty() { + vec![ChatMessage::user(&full_prompt)] + } else { + vec![ + ChatMessage::system(&system_prompt), + ChatMessage::user(&full_prompt), + ] + }; + + // Determine max_tokens from model metadata with fallback + let effective_max_tokens = match ctx.llm.model_metadata().await { + Ok(meta) => { + let from_api = meta.context_length.map(|ctx| ctx / 2).unwrap_or(max_tokens); + from_api.max(max_tokens) + } + Err(_) => max_tokens, + }; + + let request = CompletionRequest::new(messages) + .with_max_tokens(effective_max_tokens) + .with_temperature(0.3); + + let response = ctx + .llm + .complete(request) + .await + .map_err(|e| format!("LLM call failed: {e}"))?; + + let content = response.content.trim(); + let tokens_used = Some((response.input_tokens + response.output_tokens) as i32); + + // Empty content guard (same as heartbeat) + if content.is_empty() { + return if response.finish_reason == FinishReason::Length { + Err( + "LLM response truncated (finish_reason=length) with no content. \ + Model may have exhausted token budget on reasoning." + .to_string(), + ) + } else { + Err("LLM returned empty content.".to_string()) + }; + } + + // Check for the "nothing to do" sentinel + if content == "ROUTINE_OK" || content.contains("ROUTINE_OK") { + return Ok((RunStatus::Ok, None, tokens_used)); + } + + Ok((RunStatus::Attention, Some(content.to_string()), tokens_used)) +} + +/// Send a notification based on the routine's notify config and run status. +async fn send_notification( + tx: &mpsc::Sender, + notify: &NotifyConfig, + routine_name: &str, + status: RunStatus, + summary: Option<&str>, +) { + let should_notify = match status { + RunStatus::Ok => notify.on_success, + RunStatus::Attention => notify.on_attention, + RunStatus::Failed => notify.on_failure, + RunStatus::Running => false, + }; + + if !should_notify { + return; + } + + let icon = match status { + RunStatus::Ok => "✅", + RunStatus::Attention => "🔔", + RunStatus::Failed => "❌", + RunStatus::Running => "⏳", + }; + + let message = match summary { + Some(s) => format!("{} *Routine '{}'*: {}\n\n{}", icon, routine_name, status, s), + None => format!("{} *Routine '{}'*: {}", icon, routine_name, status), + }; + + let response = OutgoingResponse { + content: message, + thread_id: None, + metadata: serde_json::json!({ + "source": "routine", + "routine_name": routine_name, + "status": status.to_string(), + }), + }; + + if let Err(e) = tx.send(response).await { + tracing::error!(routine = %routine_name, "Failed to send notification: {}", e); + } +} + +/// Spawn the cron ticker background task. +pub fn spawn_cron_ticker( + engine: Arc, + interval: Duration, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + // Skip immediate first tick + ticker.tick().await; + + loop { + ticker.tick().await; + engine.check_cron_triggers().await; + } + }) +} + +fn truncate(s: &str, max: usize) -> String { + if s.len() <= max { + s.to_string() + } else { + let end = crate::util::floor_char_boundary(s, max); + format!("{}...", &s[..end]) + } +} + +#[cfg(test)] +mod tests { + use crate::agent::routine::{NotifyConfig, RunStatus}; + + #[test] + fn test_notification_gating() { + let config = NotifyConfig { + on_success: false, + on_failure: true, + on_attention: true, + ..Default::default() + }; + + // on_success = false means Ok status should not notify + assert!(!config.on_success); + assert!(config.on_failure); + assert!(config.on_attention); + } + + #[test] + fn test_run_status_icons() { + // Just verify the mapping doesn't panic + for status in [ + RunStatus::Ok, + RunStatus::Attention, + RunStatus::Failed, + RunStatus::Running, + ] { + let _ = status.to_string(); + } + } +} diff --git a/src/agent/scheduler.rs b/src/agent/scheduler.rs index a88df10a..a665c2af 100644 --- a/src/agent/scheduler.rs +++ b/src/agent/scheduler.rs @@ -12,8 +12,8 @@ use crate::agent::task::{Task, TaskContext, TaskOutput}; use crate::agent::worker::{Worker, WorkerDeps}; use crate::config::AgentConfig; use crate::context::{ContextManager, JobContext, JobState}; +use crate::db::Database; use crate::error::{Error, JobError}; -use crate::history::Store; use crate::llm::LlmProvider; use crate::safety::SafetyLayer; use crate::tools::ToolRegistry; @@ -48,7 +48,7 @@ pub struct Scheduler { llm: Arc, safety: Arc, tools: Arc, - store: Option>, + store: Option>, /// Running jobs (main LLM-driven jobs). jobs: Arc>>, /// Running sub-tasks (tool executions, background tasks). @@ -63,7 +63,7 @@ impl Scheduler { llm: Arc, safety: Arc, tools: Arc, - store: Option>, + store: Option>, ) -> Self { Self { config, @@ -79,63 +79,63 @@ impl Scheduler { /// Schedule a job for execution. pub async fn schedule(&self, job_id: Uuid) -> Result<(), JobError> { - // Check if already scheduled - if self.jobs.read().await.contains_key(&job_id) { - return Ok(()); - } + // Hold write lock for the entire check-insert sequence to prevent + // TOCTOU races where two concurrent calls both pass the checks. + { + let mut jobs = self.jobs.write().await; - // Check capacity - let current_count = self.jobs.read().await.len(); - if current_count >= self.config.max_parallel_jobs { - return Err(JobError::MaxJobsExceeded { - max: self.config.max_parallel_jobs, - }); - } - - // Transition job to in_progress - self.context_manager - .update_context(job_id, |ctx| { - ctx.transition_to( - JobState::InProgress, - Some("Scheduled for execution".to_string()), - ) - }) - .await? - .map_err(|s| JobError::ContextError { - id: job_id, - reason: s, - })?; - - // Create worker channel - let (tx, rx) = mpsc::channel(16); - - // Create worker with shared dependencies - let deps = WorkerDeps { - context_manager: self.context_manager.clone(), - llm: self.llm.clone(), - safety: self.safety.clone(), - tools: self.tools.clone(), - store: self.store.clone(), - timeout: self.config.job_timeout, - use_planning: self.config.use_planning, - }; - let worker = Worker::new(job_id, deps); - - // Spawn worker task - let handle = tokio::spawn(async move { - if let Err(e) = worker.run(rx).await { - tracing::error!("Worker for job {} failed: {}", job_id, e); + if jobs.contains_key(&job_id) { + return Ok(()); } - }); - // Start the worker - let _ = tx.send(WorkerMessage::Start).await; + if jobs.len() >= self.config.max_parallel_jobs { + return Err(JobError::MaxJobsExceeded { + max: self.config.max_parallel_jobs, + }); + } - // Store the scheduled job - self.jobs - .write() - .await - .insert(job_id, ScheduledJob { handle, tx }); + // Transition job to in_progress + self.context_manager + .update_context(job_id, |ctx| { + ctx.transition_to( + JobState::InProgress, + Some("Scheduled for execution".to_string()), + ) + }) + .await? + .map_err(|s| JobError::ContextError { + id: job_id, + reason: s, + })?; + + // Create worker channel + let (tx, rx) = mpsc::channel(16); + + // Create worker with shared dependencies + let deps = WorkerDeps { + context_manager: self.context_manager.clone(), + llm: self.llm.clone(), + safety: self.safety.clone(), + tools: self.tools.clone(), + store: self.store.clone(), + timeout: self.config.job_timeout, + use_planning: self.config.use_planning, + }; + let worker = Worker::new(job_id, deps); + + // Spawn worker task + let handle = tokio::spawn(async move { + if let Err(e) = worker.run(rx).await { + tracing::error!("Worker for job {} failed: {}", job_id, e); + } + }); + + // Start the worker + let _ = tx.send(WorkerMessage::Start).await; + + // Insert while still holding the write lock + jobs.insert(job_id, ScheduledJob { handle, tx }); + } // Cleanup task for this job to avoid capacity leaks let jobs = Arc::clone(&self.jobs); @@ -373,23 +373,23 @@ impl Scheduler { .into()); } - // Execute with timeout - let result = tokio::time::timeout(Duration::from_secs(60), async { - tool.execute(params, &job_ctx).await - }) - .await - .map_err(|_| { - Error::Tool(crate::error::ToolError::Timeout { - name: tool_name.to_string(), - timeout: Duration::from_secs(60), - }) - })? - .map_err(|e| { - Error::Tool(crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: e.to_string(), - }) - })?; + // Execute with per-tool timeout + let tool_timeout = tool.execution_timeout(); + let result = + tokio::time::timeout(tool_timeout, async { tool.execute(params, &job_ctx).await }) + .await + .map_err(|_| { + Error::Tool(crate::error::ToolError::Timeout { + name: tool_name.to_string(), + timeout: tool_timeout, + }) + })? + .map_err(|e| { + Error::Tool(crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: e.to_string(), + }) + })?; Ok(TaskOutput::new(result.result, start.elapsed())) } diff --git a/src/agent/self_repair.rs b/src/agent/self_repair.rs index 4d514405..ee7b2a4c 100644 --- a/src/agent/self_repair.rs +++ b/src/agent/self_repair.rs @@ -8,8 +8,8 @@ use chrono::{DateTime, Utc}; use uuid::Uuid; use crate::context::{ContextManager, JobState}; +use crate::db::Database; use crate::error::RepairError; -use crate::history::Store; use crate::tools::{BuildRequirement, Language, SoftwareBuilder, SoftwareType, ToolRegistry}; /// A job that has been detected as stuck. @@ -69,7 +69,7 @@ pub struct DefaultSelfRepair { #[allow(dead_code)] // Will be used for time-based stuck detection stuck_threshold: Duration, max_repair_attempts: u32, - store: Option>, + store: Option>, builder: Option>, #[allow(dead_code)] // Will be used for tool hot-reload after repair tools: Option>, @@ -94,7 +94,7 @@ impl DefaultSelfRepair { /// Add a Store for tool failure tracking. #[allow(dead_code)] // Public API for configuring repair with persistence - pub fn with_store(mut self, store: Arc) -> Self { + pub fn with_store(mut self, store: Arc) -> Self { self.store = Some(store); self } @@ -119,25 +119,25 @@ impl SelfRepair for DefaultSelfRepair { let mut stuck_jobs = Vec::new(); for job_id in stuck_ids { - if let Ok(ctx) = self.context_manager.get_context(job_id).await { - if ctx.state == JobState::Stuck { - let stuck_duration = ctx - .started_at - .map(|start| { - let now = Utc::now(); - let duration = now.signed_duration_since(start); - Duration::from_secs(duration.num_seconds().max(0) as u64) - }) - .unwrap_or_default(); + if let Ok(ctx) = self.context_manager.get_context(job_id).await + && ctx.state == JobState::Stuck + { + let stuck_duration = ctx + .started_at + .map(|start| { + let now = Utc::now(); + let duration = now.signed_duration_since(start); + Duration::from_secs(duration.num_seconds().max(0) as u64) + }) + .unwrap_or_default(); - stuck_jobs.push(StuckJob { - job_id, - last_activity: ctx.started_at.unwrap_or(ctx.created_at), - stuck_duration, - last_error: None, - repair_attempts: ctx.repair_attempts, - }); - } + stuck_jobs.push(StuckJob { + job_id, + last_activity: ctx.started_at.unwrap_or(ctx.created_at), + stuck_duration, + last_error: None, + repair_attempts: ctx.repair_attempts, + }); } } diff --git a/src/agent/session.rs b/src/agent/session.rs index a40d3fbc..e77149ec 100644 --- a/src/agent/session.rs +++ b/src/agent/session.rs @@ -173,6 +173,10 @@ pub struct Thread { /// Pending auth token request (thread is in auth mode). #[serde(default)] pub pending_auth: Option, + /// Last NEAR AI response ID for response chaining. Persisted to DB + /// metadata so we can resume chaining across restarts. + #[serde(default)] + pub last_response_id: Option, } impl Thread { @@ -189,6 +193,24 @@ impl Thread { metadata: serde_json::Value::Null, pending_approval: None, pending_auth: None, + last_response_id: None, + } + } + + /// Create a thread with a specific ID (for DB hydration). + pub fn with_id(id: Uuid, session_id: Uuid) -> Self { + let now = Utc::now(); + Self { + id, + session_id, + state: ThreadState::Idle, + turns: Vec::new(), + created_at: now, + updated_at: now, + metadata: serde_json::Value::Null, + pending_approval: None, + pending_auth: None, + last_response_id: None, } } @@ -324,11 +346,11 @@ impl Thread { let mut turn = Turn::new(turn_number, &msg.content); // Check if next is assistant response - if let Some(next) = iter.peek() { - if next.role == crate::llm::Role::Assistant { - let response = iter.next().expect("peeked"); - turn.complete(&response.content); - } + if let Some(next) = iter.peek() + && next.role == crate::llm::Role::Assistant + { + let response = iter.next().expect("peeked"); + turn.complete(&response.content); } self.turns.push(turn); @@ -593,4 +615,386 @@ mod tests { let restored: Thread = serde_json::from_str(&json).expect("should deserialize"); assert!(restored.pending_auth.is_none()); } + + #[test] + fn test_thread_with_id() { + let specific_id = Uuid::new_v4(); + let session_id = Uuid::new_v4(); + let thread = Thread::with_id(specific_id, session_id); + + assert_eq!(thread.id, specific_id); + assert_eq!(thread.session_id, session_id); + assert_eq!(thread.state, ThreadState::Idle); + assert!(thread.turns.is_empty()); + } + + #[test] + fn test_thread_with_id_restore_messages() { + let thread_id = Uuid::new_v4(); + let session_id = Uuid::new_v4(); + let mut thread = Thread::with_id(thread_id, session_id); + + let messages = vec![ + ChatMessage::user("Hello from DB"), + ChatMessage::assistant("Restored response"), + ]; + thread.restore_from_messages(messages); + + assert_eq!(thread.id, thread_id); + assert_eq!(thread.turns.len(), 1); + assert_eq!(thread.turns[0].user_input, "Hello from DB"); + assert_eq!( + thread.turns[0].response, + Some("Restored response".to_string()) + ); + } + + #[test] + fn test_restore_from_messages_empty() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Add a turn first, then restore with empty vec + thread.start_turn("hello"); + thread.complete_turn("hi"); + assert_eq!(thread.turns.len(), 1); + + thread.restore_from_messages(Vec::new()); + + // Should clear all turns and stay idle + assert!(thread.turns.is_empty()); + assert_eq!(thread.state, ThreadState::Idle); + } + + #[test] + fn test_restore_from_messages_only_assistant_messages() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Only assistant messages (no user messages to anchor turns) + let messages = vec![ + ChatMessage::assistant("I'm here"), + ChatMessage::assistant("Still here"), + ]; + + thread.restore_from_messages(messages); + + // Assistant-only messages have no user turn to attach to, so + // they should be skipped entirely. + assert!(thread.turns.is_empty()); + } + + #[test] + fn test_restore_from_messages_multiple_user_messages_in_a_row() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Two user messages with no assistant response between them + let messages = vec![ + ChatMessage::user("first"), + ChatMessage::user("second"), + ChatMessage::assistant("reply to second"), + ]; + + thread.restore_from_messages(messages); + + // First user message becomes a turn with no response, + // second user message pairs with the assistant response. + assert_eq!(thread.turns.len(), 2); + assert_eq!(thread.turns[0].user_input, "first"); + assert!(thread.turns[0].response.is_none()); + assert_eq!(thread.turns[1].user_input, "second"); + assert_eq!( + thread.turns[1].response, + Some("reply to second".to_string()) + ); + } + + #[test] + fn test_thread_switch() { + let mut session = Session::new("user-1"); + + let t1_id = session.create_thread().id; + let t2_id = session.create_thread().id; + + // After creating two threads, active should be the last one + assert_eq!(session.active_thread, Some(t2_id)); + + // Switch back to the first + assert!(session.switch_thread(t1_id)); + assert_eq!(session.active_thread, Some(t1_id)); + + // Switching to a nonexistent thread should fail + let fake_id = Uuid::new_v4(); + assert!(!session.switch_thread(fake_id)); + // Active thread should remain unchanged + assert_eq!(session.active_thread, Some(t1_id)); + } + + #[test] + fn test_get_or_create_thread_idempotent() { + let mut session = Session::new("user-1"); + + let tid1 = session.get_or_create_thread().id; + let tid2 = session.get_or_create_thread().id; + + // Should return the same thread (not create a new one each time) + assert_eq!(tid1, tid2); + assert_eq!(session.threads.len(), 1); + } + + #[test] + fn test_truncate_turns() { + let mut thread = Thread::new(Uuid::new_v4()); + + for i in 0..5 { + thread.start_turn(format!("msg-{}", i)); + thread.complete_turn(format!("resp-{}", i)); + } + assert_eq!(thread.turns.len(), 5); + + thread.truncate_turns(3); + assert_eq!(thread.turns.len(), 3); + + // Should keep the most recent turns + assert_eq!(thread.turns[0].user_input, "msg-2"); + assert_eq!(thread.turns[1].user_input, "msg-3"); + assert_eq!(thread.turns[2].user_input, "msg-4"); + + // Turn numbers should be re-indexed + assert_eq!(thread.turns[0].turn_number, 0); + assert_eq!(thread.turns[1].turn_number, 1); + assert_eq!(thread.turns[2].turn_number, 2); + } + + #[test] + fn test_truncate_turns_noop_when_fewer() { + let mut thread = Thread::new(Uuid::new_v4()); + + thread.start_turn("only one"); + thread.complete_turn("response"); + + thread.truncate_turns(10); + assert_eq!(thread.turns.len(), 1); + assert_eq!(thread.turns[0].user_input, "only one"); + } + + #[test] + fn test_thread_interrupt_and_resume() { + let mut thread = Thread::new(Uuid::new_v4()); + + thread.start_turn("do something"); + assert_eq!(thread.state, ThreadState::Processing); + + thread.interrupt(); + assert_eq!(thread.state, ThreadState::Interrupted); + + let last_turn = thread.last_turn().unwrap(); + assert_eq!(last_turn.state, TurnState::Interrupted); + assert!(last_turn.completed_at.is_some()); + + thread.resume(); + assert_eq!(thread.state, ThreadState::Idle); + } + + #[test] + fn test_resume_only_from_interrupted() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Idle thread: resume should be a no-op + assert_eq!(thread.state, ThreadState::Idle); + thread.resume(); + assert_eq!(thread.state, ThreadState::Idle); + + // Processing thread: resume should not change state + thread.start_turn("work"); + assert_eq!(thread.state, ThreadState::Processing); + thread.resume(); + assert_eq!(thread.state, ThreadState::Processing); + } + + #[test] + fn test_turn_fail() { + let mut thread = Thread::new(Uuid::new_v4()); + + thread.start_turn("risky operation"); + thread.fail_turn("connection timed out"); + + assert_eq!(thread.state, ThreadState::Idle); + + let turn = thread.last_turn().unwrap(); + assert_eq!(turn.state, TurnState::Failed); + assert_eq!(turn.error, Some("connection timed out".to_string())); + assert!(turn.response.is_none()); + assert!(turn.completed_at.is_some()); + } + + #[test] + fn test_messages_with_incomplete_last_turn() { + let mut thread = Thread::new(Uuid::new_v4()); + + thread.start_turn("first"); + thread.complete_turn("first reply"); + thread.start_turn("second (in progress)"); + + let messages = thread.messages(); + // Should have 3 messages: user, assistant, user (no assistant for in-progress) + assert_eq!(messages.len(), 3); + assert_eq!(messages[0].content, "first"); + assert_eq!(messages[1].content, "first reply"); + assert_eq!(messages[2].content, "second (in progress)"); + } + + #[test] + fn test_thread_serialization_round_trip() { + let mut thread = Thread::new(Uuid::new_v4()); + + thread.start_turn("hello"); + thread.complete_turn("world"); + thread.last_response_id = Some("resp_abc123".to_string()); + + let json = serde_json::to_string(&thread).unwrap(); + let restored: Thread = serde_json::from_str(&json).unwrap(); + + assert_eq!(restored.id, thread.id); + assert_eq!(restored.session_id, thread.session_id); + assert_eq!(restored.turns.len(), 1); + assert_eq!(restored.turns[0].user_input, "hello"); + assert_eq!(restored.turns[0].response, Some("world".to_string())); + assert_eq!(restored.last_response_id, Some("resp_abc123".to_string())); + } + + #[test] + fn test_session_serialization_round_trip() { + let mut session = Session::new("user-ser"); + session.create_thread(); + session.auto_approve_tool("echo"); + + let json = serde_json::to_string(&session).unwrap(); + let restored: Session = serde_json::from_str(&json).unwrap(); + + assert_eq!(restored.user_id, "user-ser"); + assert_eq!(restored.threads.len(), 1); + assert!(restored.is_tool_auto_approved("echo")); + assert!(!restored.is_tool_auto_approved("shell")); + } + + #[test] + fn test_auto_approved_tools() { + let mut session = Session::new("user-1"); + + assert!(!session.is_tool_auto_approved("shell")); + session.auto_approve_tool("shell"); + assert!(session.is_tool_auto_approved("shell")); + + // Idempotent + session.auto_approve_tool("shell"); + assert_eq!(session.auto_approved_tools.len(), 1); + } + + #[test] + fn test_turn_tool_call_error() { + let mut turn = Turn::new(0, "test"); + turn.record_tool_call("http", serde_json::json!({"url": "example.com"})); + turn.record_tool_error("timeout"); + + assert_eq!(turn.tool_calls.len(), 1); + assert_eq!(turn.tool_calls[0].error, Some("timeout".to_string())); + assert!(turn.tool_calls[0].result.is_none()); + } + + #[test] + fn test_turn_number_increments() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Before any turns, turn_number() is 1 (1-indexed for display) + assert_eq!(thread.turn_number(), 1); + + thread.start_turn("first"); + thread.complete_turn("done"); + assert_eq!(thread.turn_number(), 2); + + thread.start_turn("second"); + assert_eq!(thread.turn_number(), 3); + } + + #[test] + fn test_complete_turn_on_empty_thread() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Completing a turn when there are no turns should be a safe no-op + thread.complete_turn("phantom response"); + assert_eq!(thread.state, ThreadState::Idle); + assert!(thread.turns.is_empty()); + } + + #[test] + fn test_fail_turn_on_empty_thread() { + let mut thread = Thread::new(Uuid::new_v4()); + + // Failing a turn when there are no turns should be a safe no-op + thread.fail_turn("phantom error"); + assert_eq!(thread.state, ThreadState::Idle); + assert!(thread.turns.is_empty()); + } + + #[test] + fn test_pending_approval_flow() { + let mut thread = Thread::new(Uuid::new_v4()); + + let approval = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: "shell".to_string(), + parameters: serde_json::json!({"command": "rm -rf /"}), + description: "dangerous command".to_string(), + tool_call_id: "call_123".to_string(), + context_messages: vec![ChatMessage::user("do it")], + }; + + thread.await_approval(approval); + assert_eq!(thread.state, ThreadState::AwaitingApproval); + assert!(thread.pending_approval.is_some()); + + let taken = thread.take_pending_approval(); + assert!(taken.is_some()); + assert_eq!(taken.unwrap().tool_name, "shell"); + assert!(thread.pending_approval.is_none()); + } + + #[test] + fn test_clear_pending_approval() { + let mut thread = Thread::new(Uuid::new_v4()); + + let approval = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: "http".to_string(), + parameters: serde_json::json!({}), + description: "test".to_string(), + tool_call_id: "call_456".to_string(), + context_messages: vec![], + }; + + thread.await_approval(approval); + thread.clear_pending_approval(); + + assert_eq!(thread.state, ThreadState::Idle); + assert!(thread.pending_approval.is_none()); + } + + #[test] + fn test_active_thread_accessors() { + let mut session = Session::new("user-1"); + + assert!(session.active_thread().is_none()); + assert!(session.active_thread_mut().is_none()); + + let tid = session.create_thread().id; + + assert!(session.active_thread().is_some()); + assert_eq!(session.active_thread().unwrap().id, tid); + + // Mutably modify through accessor + session.active_thread_mut().unwrap().start_turn("test"); + assert_eq!( + session.active_thread().unwrap().state, + ThreadState::Processing + ); + } } diff --git a/src/agent/session_manager.rs b/src/agent/session_manager.rs index dcf46343..db0be886 100644 --- a/src/agent/session_manager.rs +++ b/src/agent/session_manager.rs @@ -110,6 +110,41 @@ impl SessionManager { (session, thread_id) } + /// Register a hydrated thread so subsequent `resolve_thread` calls find it. + /// + /// Inserts into the thread_map and creates an undo manager for the thread. + pub async fn register_thread( + &self, + user_id: &str, + channel: &str, + thread_id: Uuid, + session: Arc>, + ) { + let key = ThreadKey { + user_id: user_id.to_string(), + channel: channel.to_string(), + external_thread_id: Some(thread_id.to_string()), + }; + + { + let mut thread_map = self.thread_map.write().await; + thread_map.insert(key, thread_id); + } + + { + let mut undo_managers = self.undo_managers.write().await; + undo_managers + .entry(thread_id) + .or_insert_with(|| Arc::new(Mutex::new(UndoManager::new()))); + } + + // Ensure the session is tracked + { + let mut sessions = self.sessions.write().await; + sessions.entry(user_id.to_string()).or_insert(session); + } + } + /// Get undo manager for a thread. pub async fn get_undo_manager(&self, thread_id: Uuid) -> Arc> { // Fast path @@ -164,10 +199,10 @@ impl SessionManager { { let sessions = self.sessions.read().await; for user_id in &stale_users { - if let Some(session) = sessions.get(user_id) { - if let Ok(sess) = session.try_lock() { - stale_thread_ids.extend(sess.threads.keys()); - } + if let Some(session) = sessions.get(user_id) + && let Ok(sess) = session.try_lock() + { + stale_thread_ids.extend(sess.threads.keys()); } } } @@ -296,4 +331,344 @@ mod tests { .await; assert_eq!(pruned, 0); } + + #[tokio::test] + async fn test_register_thread() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let thread_id = Uuid::new_v4(); + + // Create a session with a hydrated thread + let session = Arc::new(Mutex::new(Session::new("user-hydrate"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(thread_id, sess.id); + sess.threads.insert(thread_id, thread); + sess.active_thread = Some(thread_id); + } + + // Register the thread + manager + .register_thread("user-hydrate", "gateway", thread_id, Arc::clone(&session)) + .await; + + // resolve_thread should find it (using the UUID as external_thread_id) + let (resolved_session, resolved_tid) = manager + .resolve_thread("user-hydrate", "gateway", Some(&thread_id.to_string())) + .await; + assert_eq!(resolved_tid, thread_id); + + // Should be the same session object + let sess = resolved_session.lock().await; + assert!(sess.threads.contains_key(&thread_id)); + } + + #[tokio::test] + async fn test_resolve_thread_with_explicit_external_id() { + let manager = SessionManager::new(); + + // Two calls with the same explicit external thread ID should resolve + // to the same internal thread. + let (_, t1) = manager + .resolve_thread("user-1", "gateway", Some("ext-abc")) + .await; + let (_, t2) = manager + .resolve_thread("user-1", "gateway", Some("ext-abc")) + .await; + assert_eq!(t1, t2); + + // A different external ID on the same channel/user gets a new thread. + let (_, t3) = manager + .resolve_thread("user-1", "gateway", Some("ext-xyz")) + .await; + assert_ne!(t1, t3); + } + + #[tokio::test] + async fn test_resolve_thread_none_vs_some_external_id() { + let manager = SessionManager::new(); + + // None external_thread_id is a distinct key from Some("ext-1"). + let (_, t_none) = manager.resolve_thread("user-1", "cli", None).await; + let (_, t_some) = manager.resolve_thread("user-1", "cli", Some("ext-1")).await; + assert_ne!(t_none, t_some); + } + + #[tokio::test] + async fn test_resolve_thread_different_users_isolated() { + let manager = SessionManager::new(); + + let (_, t1) = manager + .resolve_thread("user-a", "gateway", Some("same-ext")) + .await; + let (_, t2) = manager + .resolve_thread("user-b", "gateway", Some("same-ext")) + .await; + + // Same channel + same external ID but different users = different threads + assert_ne!(t1, t2); + } + + #[tokio::test] + async fn test_resolve_thread_different_channels_isolated() { + let manager = SessionManager::new(); + + let (_, t1) = manager + .resolve_thread("user-1", "gateway", Some("thread-x")) + .await; + let (_, t2) = manager + .resolve_thread("user-1", "telegram", Some("thread-x")) + .await; + + // Same user + same external ID but different channels = different threads + assert_ne!(t1, t2); + } + + #[tokio::test] + async fn test_resolve_thread_stale_mapping_creates_new_thread() { + let manager = SessionManager::new(); + + // Create a thread normally + let (session, original_tid) = manager + .resolve_thread("user-1", "gateway", Some("ext-1")) + .await; + + // Simulate the thread being removed from the session (e.g. pruned) + { + let mut sess = session.lock().await; + sess.threads.remove(&original_tid); + } + + // Next resolve should detect the stale mapping and create a fresh thread + let (_, new_tid) = manager + .resolve_thread("user-1", "gateway", Some("ext-1")) + .await; + assert_ne!(original_tid, new_tid); + + // The new thread should actually exist in the session + let sess = session.lock().await; + assert!(sess.threads.contains_key(&new_tid)); + } + + #[tokio::test] + async fn test_register_thread_preserves_uuid_on_resolve() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let known_uuid = Uuid::new_v4(); + + let session = Arc::new(Mutex::new(Session::new("user-web"))); + let session_id = { + let sess = session.lock().await; + sess.id + }; + + // Simulate hydration: create thread with a known UUID + { + let mut sess = session.lock().await; + let thread = Thread::with_id(known_uuid, session_id); + sess.threads.insert(known_uuid, thread); + } + + // Register it + manager + .register_thread("user-web", "gateway", known_uuid, Arc::clone(&session)) + .await; + + // resolve_thread with UUID as external_thread_id MUST return the same UUID, + // not mint a new one (this was the root cause of the "wrong conversation" bug) + let (_, resolved) = manager + .resolve_thread("user-web", "gateway", Some(&known_uuid.to_string())) + .await; + assert_eq!(resolved, known_uuid); + } + + #[tokio::test] + async fn test_register_thread_idempotent() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let tid = Uuid::new_v4(); + + let session = Arc::new(Mutex::new(Session::new("user-idem"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(tid, sess.id); + sess.threads.insert(tid, thread); + } + + // Register twice + manager + .register_thread("user-idem", "gateway", tid, Arc::clone(&session)) + .await; + manager + .register_thread("user-idem", "gateway", tid, Arc::clone(&session)) + .await; + + // Should still resolve to the same thread + let (_, resolved) = manager + .resolve_thread("user-idem", "gateway", Some(&tid.to_string())) + .await; + assert_eq!(resolved, tid); + } + + #[tokio::test] + async fn test_register_thread_creates_undo_manager() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let tid = Uuid::new_v4(); + + let session = Arc::new(Mutex::new(Session::new("user-undo"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(tid, sess.id); + sess.threads.insert(tid, thread); + } + + manager + .register_thread("user-undo", "gateway", tid, Arc::clone(&session)) + .await; + + // Undo manager should exist for the registered thread + let undo = manager.get_undo_manager(tid).await; + let undo2 = manager.get_undo_manager(tid).await; + assert!(Arc::ptr_eq(&undo, &undo2)); + } + + #[tokio::test] + async fn test_register_thread_stores_session() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let tid = Uuid::new_v4(); + + let session = Arc::new(Mutex::new(Session::new("user-new"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(tid, sess.id); + sess.threads.insert(tid, thread); + } + + // The user has no session yet in the manager + { + let sessions = manager.sessions.read().await; + assert!(!sessions.contains_key("user-new")); + } + + manager + .register_thread("user-new", "gateway", tid, Arc::clone(&session)) + .await; + + // Now the session should be tracked + { + let sessions = manager.sessions.read().await; + assert!(sessions.contains_key("user-new")); + } + } + + #[tokio::test] + async fn test_multiple_threads_per_user() { + let manager = SessionManager::new(); + + let (_, t1) = manager + .resolve_thread("user-1", "gateway", Some("thread-a")) + .await; + let (_, t2) = manager + .resolve_thread("user-1", "gateway", Some("thread-b")) + .await; + let (session, t3) = manager + .resolve_thread("user-1", "gateway", Some("thread-c")) + .await; + + // All three should be distinct + assert_ne!(t1, t2); + assert_ne!(t2, t3); + assert_ne!(t1, t3); + + // All three should exist in the same session + let sess = session.lock().await; + assert!(sess.threads.contains_key(&t1)); + assert!(sess.threads.contains_key(&t2)); + assert!(sess.threads.contains_key(&t3)); + } + + #[tokio::test] + async fn test_prune_cleans_thread_map_and_undo_managers() { + let manager = SessionManager::new(); + + let (stale_session, stale_tid) = manager.resolve_thread("user-stale", "cli", None).await; + + // Backdate the session + { + let mut sess = stale_session.lock().await; + sess.last_active_at = chrono::Utc::now() - chrono::TimeDelta::seconds(86400 * 30); + } + + // Verify thread_map and undo_managers have entries + { + let tm = manager.thread_map.read().await; + assert!(!tm.is_empty()); + } + { + let um = manager.undo_managers.read().await; + assert!(um.contains_key(&stale_tid)); + } + + let pruned = manager + .prune_stale_sessions(std::time::Duration::from_secs(86400 * 7)) + .await; + assert_eq!(pruned, 1); + + // Thread map and undo managers should be cleaned up + { + let tm = manager.thread_map.read().await; + assert!(tm.is_empty()); + } + { + let um = manager.undo_managers.read().await; + assert!(!um.contains_key(&stale_tid)); + } + } + + #[tokio::test] + async fn test_resolve_thread_active_thread_set() { + let manager = SessionManager::new(); + + let (session, thread_id) = manager + .resolve_thread("user-1", "gateway", Some("ext-1")) + .await; + + // The resolved thread should be set as the active thread + let sess = session.lock().await; + assert_eq!(sess.active_thread, Some(thread_id)); + } + + #[tokio::test] + async fn test_register_then_resolve_different_channel_creates_new() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let tid = Uuid::new_v4(); + + let session = Arc::new(Mutex::new(Session::new("user-cross"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(tid, sess.id); + sess.threads.insert(tid, thread); + } + + // Register on "gateway" channel + manager + .register_thread("user-cross", "gateway", tid, Arc::clone(&session)) + .await; + + // Resolve on a different channel with the same UUID string should NOT + // find the registered thread (channel is part of the key) + let (_, resolved) = manager + .resolve_thread("user-cross", "telegram", Some(&tid.to_string())) + .await; + assert_ne!(resolved, tid); + } } diff --git a/src/agent/submission.rs b/src/agent/submission.rs index 7a28356c..a2b6b4d7 100644 --- a/src/agent/submission.rs +++ b/src/agent/submission.rs @@ -43,6 +43,49 @@ impl SubmissionParser { if lower == "/thread new" || lower == "/new" { return Submission::NewThread; } + // System commands (bypass thread-state checks) + if lower == "/help" || lower == "/?" { + return Submission::SystemCommand { + command: "help".to_string(), + args: vec![], + }; + } + if lower == "/version" { + return Submission::SystemCommand { + command: "version".to_string(), + args: vec![], + }; + } + if lower == "/tools" { + return Submission::SystemCommand { + command: "tools".to_string(), + args: vec![], + }; + } + if lower == "/ping" { + return Submission::SystemCommand { + command: "ping".to_string(), + args: vec![], + }; + } + if lower == "/debug" { + return Submission::SystemCommand { + command: "debug".to_string(), + args: vec![], + }; + } + if lower.starts_with("/model") { + let args: Vec = trimmed + .split_whitespace() + .skip(1) + .map(|s| s.to_string()) + .collect(); + return Submission::SystemCommand { + command: "model".to_string(), + args, + }; + } + if lower == "/quit" || lower == "/exit" || lower == "/shutdown" { return Submission::Quit; } @@ -50,27 +93,26 @@ impl SubmissionParser { // /thread - switch thread if let Some(rest) = lower.strip_prefix("/thread ") { let rest = rest.trim(); - if rest != "new" { - if let Ok(id) = Uuid::parse_str(rest) { - return Submission::SwitchThread { thread_id: id }; - } + if rest != "new" + && let Ok(id) = Uuid::parse_str(rest) + { + return Submission::SwitchThread { thread_id: id }; } } // /resume - resume from checkpoint - if let Some(rest) = lower.strip_prefix("/resume ") { - if let Ok(id) = Uuid::parse_str(rest.trim()) { - return Submission::Resume { checkpoint_id: id }; - } + if let Some(rest) = lower.strip_prefix("/resume ") + && let Ok(id) = Uuid::parse_str(rest.trim()) + { + return Submission::Resume { checkpoint_id: id }; } // Try structured JSON approval (from web gateway's /api/chat/approval endpoint) - if trimmed.starts_with('{') { - if let Ok(submission) = serde_json::from_str::(trimmed) { - if matches!(submission, Submission::ExecApproval { .. }) { - return submission; - } - } + if trimmed.starts_with('{') + && let Ok(submission) = serde_json::from_str::(trimmed) + && matches!(submission, Submission::ExecApproval { .. }) + { + return submission; } // Approval responses (simple yes/no/always for pending approvals) @@ -172,6 +214,15 @@ pub enum Submission { /// Quit the agent. Bypasses thread-state checks. Quit, + + /// System command (help, model, version, tools, ping, debug). + /// Bypasses thread-state checks and safety validation. + SystemCommand { + /// The command name (e.g. "help", "model", "version"). + command: String, + /// Arguments to the command. + args: Vec, + }, } impl Submission { @@ -238,6 +289,7 @@ impl Submission { | Self::Heartbeat | Self::Summarize | Self::Suggest + | Self::SystemCommand { .. } ) } } @@ -504,6 +556,84 @@ mod tests { ); } + #[test] + fn test_parser_system_command_help() { + let submission = SubmissionParser::parse("/help"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "help" && args.is_empty()) + ); + + let submission = SubmissionParser::parse("/?"); + assert!( + matches!(submission, Submission::SystemCommand { command, .. } if command == "help") + ); + + let submission = SubmissionParser::parse("/HELP"); + assert!( + matches!(submission, Submission::SystemCommand { command, .. } if command == "help") + ); + } + + #[test] + fn test_parser_system_command_model() { + // No args: show current model + let submission = SubmissionParser::parse("/model"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "model" && args.is_empty()) + ); + + // With args: switch model + let submission = SubmissionParser::parse("/model gpt-4o"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "model" && args == vec!["gpt-4o"]) + ); + + // Case insensitive command, preserves arg case + let submission = SubmissionParser::parse("/MODEL Claude-3.5"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "model" && args == vec!["Claude-3.5"]) + ); + } + + #[test] + fn test_parser_system_command_version() { + let submission = SubmissionParser::parse("/version"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "version" && args.is_empty()) + ); + } + + #[test] + fn test_parser_system_command_tools() { + let submission = SubmissionParser::parse("/tools"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "tools" && args.is_empty()) + ); + } + + #[test] + fn test_parser_system_command_ping() { + let submission = SubmissionParser::parse("/ping"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "ping" && args.is_empty()) + ); + } + + #[test] + fn test_parser_system_command_debug() { + let submission = SubmissionParser::parse("/debug"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "debug" && args.is_empty()) + ); + } + + #[test] + fn test_parser_system_command_is_control() { + let submission = SubmissionParser::parse("/help"); + assert!(submission.is_control()); + assert!(!submission.starts_turn()); + } + #[test] fn test_parser_quit() { assert!(matches!(SubmissionParser::parse("/quit"), Submission::Quit)); diff --git a/src/agent/worker.rs b/src/agent/worker.rs index c05ece63..565a3d22 100644 --- a/src/agent/worker.rs +++ b/src/agent/worker.rs @@ -10,8 +10,8 @@ use uuid::Uuid; use crate::agent::scheduler::WorkerMessage; use crate::agent::task::TaskOutput; use crate::context::{ContextManager, JobState}; +use crate::db::Database; use crate::error::Error; -use crate::history::Store; use crate::llm::{ ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection, }; @@ -28,7 +28,7 @@ pub struct WorkerDeps { pub llm: Arc, pub safety: Arc, pub tools: Arc, - pub store: Option>, + pub store: Option>, pub timeout: Duration, pub use_planning: bool, } @@ -67,7 +67,7 @@ impl Worker { &self.deps.tools } - fn store(&self) -> Option<&Arc> { + fn store(&self) -> Option<&Arc> { self.deps.store.as_ref() } @@ -227,11 +227,11 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } // Check for cancellation - if let Ok(ctx) = self.context_manager().get_context(self.job_id).await { - if ctx.state == JobState::Cancelled { - tracing::info!("Worker for job {} detected cancellation", self.job_id); - return Ok(()); - } + if let Ok(ctx) = self.context_manager().get_context(self.job_id).await + && ctx.state == JobState::Cancelled + { + tracing::info!("Worker for job {} detected cancellation", self.job_id); + return Ok(()); } iteration += 1; @@ -248,16 +248,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."# if selections.is_empty() { // No tools from select_tools, ask LLM directly (may still return tool calls) - let respond_result = reasoning.respond_with_tools(reason_ctx).await?; + let respond_output = reasoning.respond_with_tools(reason_ctx).await?; - match respond_result { + match respond_output.result { RespondResult::Text(response) => { - // Check for completion keywords - let response_lower = response.to_lowercase(); - if response_lower.contains("complete") - || response_lower.contains("finished") - || response_lower.contains("done") - { + // Check for explicit completion phrases. Use word-boundary + // aware checks to avoid false positives like "incomplete", + // "not done", or "unfinished". Only the LLM's own response + // (not tool output) can trigger this. + if crate::util::llm_signals_completion(&response) { self.mark_completed().await?; return Ok(()); } @@ -272,7 +271,10 @@ Report when the job is complete or if you encounter issues you cannot resolve."# )); } } - RespondResult::ToolCalls(tool_calls) => { + RespondResult::ToolCalls { + tool_calls, + content, + } => { // Model returned tool calls - execute them tracing::debug!( "Job {} respond_with_tools returned {} tool calls", @@ -280,6 +282,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tool_calls.len() ); + // Add assistant message with tool_calls (OpenAI protocol) + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls( + content, + tool_calls.clone(), + )); + for tc in tool_calls { let result = self.execute_tool(&tc.name, &tc.arguments).await; @@ -289,6 +299,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# parameters: tc.arguments.clone(), reasoning: String::new(), alternatives: vec![], + tool_call_id: tc.id.clone(), }; self.process_tool_result(reason_ctx, &selection, result) @@ -371,7 +382,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tools: Arc, context_manager: Arc, safety: Arc, - store: Option>, + store: Option>, job_id: Uuid, tool_name: &str, params: &serde_json::Value, @@ -417,14 +428,51 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .into()); } - // Execute with timeout and timing + tracing::debug!( + tool = %tool_name, + params = %params, + job = %job_id, + "Tool call started" + ); + + // Execute with per-tool timeout and timing + let tool_timeout = tool.execution_timeout(); let start = std::time::Instant::now(); - let result = tokio::time::timeout(Duration::from_secs(60), async { + let result = tokio::time::timeout(tool_timeout, async { tool.execute(params.clone(), &job_ctx).await }) .await; let elapsed = start.elapsed(); + match &result { + Ok(Ok(output)) => { + let result_str = serde_json::to_string(&output.result) + .unwrap_or_else(|_| "".to_string()); + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + result = %result_str, + "Tool call succeeded" + ); + } + Ok(Err(e)) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + error = %e, + "Tool call failed" + ); + } + Err(_) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + timeout_secs = tool_timeout.as_secs(), + "Tool call timed out" + ); + } + } + // Record action in memory and get the ActionRecord for persistence let action = match &result { Ok(Ok(output)) => { @@ -479,7 +527,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# let output = result .map_err(|_| crate::error::ToolError::Timeout { name: tool_name.to_string(), - timeout: Duration::from_secs(60), + timeout: tool_timeout, })? .map_err(|e| crate::error::ToolError::ExecutionFailed { name: tool_name.to_string(), @@ -518,17 +566,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# ); reason_ctx.messages.push(ChatMessage::tool_result( - "tool_call_id", + &selection.tool_call_id, &selection.tool_name, wrapped, )); - // Check if job is complete - if output.contains("TASK_COMPLETE") || output.contains("JOB_DONE") { - self.mark_completed().await?; - return Ok(true); - } - + // Tool output never drives job completion. A malicious tool could + // emit "TASK_COMPLETE" to force premature completion. Only the LLM's + // own structured response (in execution_loop) can mark a job done. Ok(false) } Err(e) => { @@ -553,7 +598,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } reason_ctx.messages.push(ChatMessage::tool_result( - "tool_call_id", + &selection.tool_call_id, &selection.tool_name, format!("Error: {}", e), )); @@ -603,12 +648,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .execute_tool(&action.tool_name, &action.parameters) .await; - // Create a synthetic ToolSelection for process_tool_result + // Create a synthetic ToolSelection for process_tool_result. + // Plan actions don't originate from an LLM tool_call response so + // there is no real tool_call_id; generate a unique one. let selection = ToolSelection { tool_name: action.tool_name.clone(), parameters: action.parameters.clone(), reasoning: action.reasoning.clone(), alternatives: vec![], + tool_call_id: format!("plan_{}_{}", self.job_id, i), }; // Process the result @@ -632,11 +680,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# let response = reasoning.respond(reason_ctx).await?; reason_ctx.messages.push(ChatMessage::assistant(&response)); - let response_lower = response.to_lowercase(); - if response_lower.contains("complete") - || response_lower.contains("finished") - || response_lower.contains("done") - { + if crate::util::llm_signals_completion(&response) { self.mark_completed().await?; } else { // Job not complete, could re-plan or fall back to direct selection @@ -731,3 +775,86 @@ impl From for Result { }) } } + +#[cfg(test)] +mod tests { + use crate::llm::ToolSelection; + use crate::util::llm_signals_completion; + + #[test] + fn test_tool_selection_preserves_call_id() { + let selection = ToolSelection { + tool_name: "memory_search".to_string(), + parameters: serde_json::json!({"query": "test"}), + reasoning: "Need to search memory".to_string(), + alternatives: vec![], + tool_call_id: "call_abc123".to_string(), + }; + + assert_eq!(selection.tool_call_id, "call_abc123"); + assert_ne!( + selection.tool_call_id, "tool_call_id", + "tool_call_id must not be the hardcoded placeholder string" + ); + } + + #[test] + fn test_completion_positive_signals() { + assert!(llm_signals_completion("The job is complete.")); + assert!(llm_signals_completion( + "I have completed the task successfully." + )); + assert!(llm_signals_completion("The task is done.")); + assert!(llm_signals_completion("The task is finished.")); + assert!(llm_signals_completion( + "All steps are complete and verified." + )); + assert!(llm_signals_completion( + "I've done all the work. The work is done." + )); + assert!(llm_signals_completion( + "Successfully completed the migration." + )); + } + + #[test] + fn test_completion_negative_signals_block_false_positives() { + // These contain completion keywords but also negation, should NOT trigger. + assert!(!llm_signals_completion("The task is not complete yet.")); + assert!(!llm_signals_completion("This is not done.")); + assert!(!llm_signals_completion("The work is incomplete.")); + assert!(!llm_signals_completion( + "The migration is not yet finished." + )); + assert!(!llm_signals_completion("The job isn't done yet.")); + assert!(!llm_signals_completion("This remains unfinished.")); + } + + #[test] + fn test_completion_does_not_match_bare_substrings() { + // Bare words embedded in other text should NOT trigger completion. + assert!(!llm_signals_completion( + "I need to complete more work first." + )); + assert!(!llm_signals_completion( + "Let me finish the remaining steps." + )); + assert!(!llm_signals_completion( + "I'm done analyzing, now let me fix it." + )); + assert!(!llm_signals_completion( + "I completed step 1 but step 2 remains." + )); + } + + #[test] + fn test_completion_tool_output_injection() { + // A malicious tool output echoed by the LLM should not trigger + // completion unless it forms a genuine completion phrase. + assert!(!llm_signals_completion("TASK_COMPLETE")); + assert!(!llm_signals_completion("JOB_DONE")); + assert!(!llm_signals_completion( + "The tool returned: TASK_COMPLETE signal" + )); + } +} diff --git a/src/bootstrap.rs b/src/bootstrap.rs new file mode 100644 index 00000000..6c14efdf --- /dev/null +++ b/src/bootstrap.rs @@ -0,0 +1,388 @@ +//! Bootstrap helpers for IronClaw. +//! +//! The only setting that truly needs disk persistence before the database is +//! available is `DATABASE_URL` (chicken-and-egg: can't connect to DB without +//! it). Everything else is auto-detected or read from env vars. +//! +//! File: `~/.ironclaw/.env` (standard dotenvy format) + +use std::path::PathBuf; + +/// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`. +pub fn ironclaw_env_path() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw") + .join(".env") +} + +/// Load env vars from `~/.ironclaw/.env` (in addition to the standard `.env`). +/// +/// Call this **after** `dotenvy::dotenv()` so that the standard `./.env` +/// takes priority over `~/.ironclaw/.env`. dotenvy never overwrites +/// existing env vars, so the effective priority is: +/// +/// explicit env vars > `./.env` > `~/.ironclaw/.env` +/// +/// If `~/.ironclaw/.env` doesn't exist but the legacy `bootstrap.json` does, +/// extracts `DATABASE_URL` from it and writes the `.env` file (one-time +/// upgrade from the old config format). +pub fn load_ironclaw_env() { + let path = ironclaw_env_path(); + + if !path.exists() { + // One-time upgrade: extract DATABASE_URL from legacy bootstrap.json + migrate_bootstrap_json_to_env(&path); + } + + if path.exists() { + let _ = dotenvy::from_path(&path); + } +} + +/// If `bootstrap.json` exists, pull `database_url` out of it and write `.env`. +fn migrate_bootstrap_json_to_env(env_path: &std::path::Path) { + let ironclaw_dir = env_path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + let bootstrap_path = ironclaw_dir.join("bootstrap.json"); + + if !bootstrap_path.exists() { + return; + } + + let content = match std::fs::read_to_string(&bootstrap_path) { + Ok(c) => c, + Err(_) => return, + }; + + // Minimal parse: just grab database_url from the JSON + let parsed: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(_) => return, + }; + + if let Some(url) = parsed.get("database_url").and_then(|v| v.as_str()) { + if let Some(parent) = env_path.parent() + && let Err(e) = std::fs::create_dir_all(parent) + { + eprintln!("Warning: failed to create {}: {}", parent.display(), e); + return; + } + if let Err(e) = std::fs::write(env_path, format!("DATABASE_URL=\"{}\"\n", url)) { + eprintln!("Warning: failed to migrate bootstrap.json to .env: {}", e); + return; + } + rename_to_migrated(&bootstrap_path); + eprintln!( + "Migrated DATABASE_URL from bootstrap.json to {}", + env_path.display() + ); + } +} + +/// Write `DATABASE_URL` to `~/.ironclaw/.env`. +/// +/// Creates the parent directory if it doesn't exist. +/// The value is double-quoted so that `#` (common in URL-encoded passwords) +/// and other shell-special characters are preserved by dotenvy. +pub fn save_database_url(url: &str) -> std::io::Result<()> { + let path = ironclaw_env_path(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, format!("DATABASE_URL=\"{}\"\n", url)) +} + +/// One-time migration of legacy `~/.ironclaw/settings.json` into the database. +/// +/// Only runs when a `settings.json` exists on disk AND the DB has no settings +/// yet. After the wizard writes directly to the DB, this path is only hit by +/// users upgrading from the old disk-only configuration. +/// +/// After syncing, renames `settings.json` to `.migrated` so it won't trigger again. +pub async fn migrate_disk_to_db( + store: &dyn crate::db::Database, + user_id: &str, +) -> Result<(), MigrationError> { + let ironclaw_dir = dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw"); + let legacy_settings_path = ironclaw_dir.join("settings.json"); + + if !legacy_settings_path.exists() { + tracing::debug!("No legacy settings.json found, skipping disk-to-DB migration"); + return Ok(()); + } + + // If DB already has settings, this is not a first boot, the wizard already + // wrote directly to the DB. Just clean up the stale file. + let has_settings = store.has_settings(user_id).await.map_err(|e| { + MigrationError::Database(format!("Failed to check existing settings: {}", e)) + })?; + if has_settings { + tracing::info!("DB already has settings, renaming stale settings.json"); + rename_to_migrated(&legacy_settings_path); + return Ok(()); + } + + tracing::info!("Migrating disk settings to database..."); + + // 1. Load and migrate settings.json + let settings = crate::settings::Settings::load_from(&legacy_settings_path); + let db_map = settings.to_db_map(); + if !db_map.is_empty() { + store + .set_all_settings(user_id, &db_map) + .await + .map_err(|e| { + MigrationError::Database(format!("Failed to write settings to DB: {}", e)) + })?; + tracing::info!("Migrated {} settings to database", db_map.len()); + } + + // 2. Write DATABASE_URL to ~/.ironclaw/.env + if let Some(ref url) = settings.database_url { + save_database_url(url) + .map_err(|e| MigrationError::Io(format!("Failed to write .env: {}", e)))?; + tracing::info!("Wrote DATABASE_URL to {}", ironclaw_env_path().display()); + } + + // 3. Migrate mcp-servers.json if it exists + let mcp_path = ironclaw_dir.join("mcp-servers.json"); + if mcp_path.exists() { + match std::fs::read_to_string(&mcp_path) { + Ok(content) => match serde_json::from_str::(&content) { + Ok(value) => { + store + .set_setting(user_id, "mcp_servers", &value) + .await + .map_err(|e| { + MigrationError::Database(format!( + "Failed to write MCP servers to DB: {}", + e + )) + })?; + tracing::info!("Migrated mcp-servers.json to database"); + + rename_to_migrated(&mcp_path); + } + Err(e) => { + tracing::warn!("Failed to parse mcp-servers.json: {}", e); + } + }, + Err(e) => { + tracing::warn!("Failed to read mcp-servers.json: {}", e); + } + } + } + + // 4. Migrate session.json if it exists + let session_path = ironclaw_dir.join("session.json"); + if session_path.exists() { + match std::fs::read_to_string(&session_path) { + Ok(content) => match serde_json::from_str::(&content) { + Ok(value) => { + store + .set_setting(user_id, "nearai.session", &value) + .await + .map_err(|e| { + MigrationError::Database(format!( + "Failed to write session to DB: {}", + e + )) + })?; + tracing::info!("Migrated session.json to database"); + + rename_to_migrated(&session_path); + } + Err(e) => { + tracing::warn!("Failed to parse session.json: {}", e); + } + }, + Err(e) => { + tracing::warn!("Failed to read session.json: {}", e); + } + } + } + + // 5. Rename settings.json to .migrated (don't delete, safety net) + rename_to_migrated(&legacy_settings_path); + + // 6. Clean up old bootstrap.json if it exists (superseded by .env) + let old_bootstrap = ironclaw_dir.join("bootstrap.json"); + if old_bootstrap.exists() { + rename_to_migrated(&old_bootstrap); + tracing::info!("Renamed old bootstrap.json to .migrated"); + } + + tracing::info!("Disk-to-DB migration complete"); + Ok(()) +} + +/// Rename a file to `.migrated` as a safety net. +fn rename_to_migrated(path: &std::path::Path) { + let mut migrated = path.as_os_str().to_owned(); + migrated.push(".migrated"); + if let Err(e) = std::fs::rename(path, &migrated) { + tracing::warn!("Failed to rename {} to .migrated: {}", path.display(), e); + } +} + +/// Errors that can occur during disk-to-DB migration. +#[derive(Debug, thiserror::Error)] +pub enum MigrationError { + #[error("Database error: {0}")] + Database(String), + #[error("IO error: {0}")] + Io(String), +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn test_save_and_load_database_url() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // Write in the quoted format that save_database_url uses + let url = "postgres://localhost:5432/ironclaw_test"; + std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap(); + + // Verify the content is a valid dotenv line (quoted) + let content = std::fs::read_to_string(&env_path).unwrap(); + assert_eq!( + content, + "DATABASE_URL=\"postgres://localhost:5432/ironclaw_test\"\n" + ); + + // Verify dotenvy can parse it (strips quotes automatically) + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].0, "DATABASE_URL"); + assert_eq!(parsed[0].1, url); + } + + #[test] + fn test_save_database_url_with_hash_in_password() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // URLs with # in the password are common (URL-encoded special chars). + // Without quoting, dotenvy treats # as a comment delimiter. + let url = "postgres://user:p%23ss@localhost:5432/ironclaw"; + std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap(); + + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].0, "DATABASE_URL"); + assert_eq!(parsed[0].1, url); + } + + #[test] + fn test_save_database_url_creates_parent_dirs() { + let dir = tempdir().unwrap(); + let nested = dir.path().join("deep").join("nested"); + let env_path = nested.join(".env"); + + // Parent doesn't exist yet + assert!(!nested.exists()); + + // The global function uses a fixed path, so we test the logic directly + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(&env_path, "DATABASE_URL=postgres://test\n").unwrap(); + + assert!(env_path.exists()); + let content = std::fs::read_to_string(&env_path).unwrap(); + assert!(content.contains("DATABASE_URL=postgres://test")); + } + + #[test] + fn test_ironclaw_env_path() { + let path = ironclaw_env_path(); + assert!(path.ends_with(".ironclaw/.env")); + } + + #[test] + fn test_migrate_bootstrap_json_to_env() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + let bootstrap_path = dir.path().join("bootstrap.json"); + + // Write a legacy bootstrap.json + let bootstrap_json = serde_json::json!({ + "database_url": "postgres://localhost/ironclaw_upgrade", + "database_pool_size": 5, + "secrets_master_key_source": "keychain", + "onboard_completed": true + }); + std::fs::write( + &bootstrap_path, + serde_json::to_string_pretty(&bootstrap_json).unwrap(), + ) + .unwrap(); + + assert!(!env_path.exists()); + assert!(bootstrap_path.exists()); + + // Run the migration + migrate_bootstrap_json_to_env(&env_path); + + // .env should now exist with DATABASE_URL + assert!(env_path.exists()); + let content = std::fs::read_to_string(&env_path).unwrap(); + assert_eq!( + content, + "DATABASE_URL=\"postgres://localhost/ironclaw_upgrade\"\n" + ); + + // bootstrap.json should be renamed to .migrated + assert!(!bootstrap_path.exists()); + assert!(dir.path().join("bootstrap.json.migrated").exists()); + } + + #[test] + fn test_migrate_bootstrap_json_no_database_url() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + let bootstrap_path = dir.path().join("bootstrap.json"); + + // bootstrap.json with no database_url + let bootstrap_json = serde_json::json!({ + "onboard_completed": false + }); + std::fs::write( + &bootstrap_path, + serde_json::to_string_pretty(&bootstrap_json).unwrap(), + ) + .unwrap(); + + migrate_bootstrap_json_to_env(&env_path); + + // .env should NOT be created + assert!(!env_path.exists()); + // bootstrap.json should remain (no migration happened) + assert!(bootstrap_path.exists()); + } + + #[test] + fn test_migrate_bootstrap_json_missing() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // No bootstrap.json at all + migrate_bootstrap_json_to_env(&env_path); + + // Nothing should happen + assert!(!env_path.exists()); + } +} diff --git a/src/channels/channel.rs b/src/channels/channel.rs index c5575ccd..d87c8240 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -114,6 +114,12 @@ pub enum StatusUpdate { StreamChunk(String), /// General status message. Status(String), + /// A sandbox job has started (shown as a clickable card in the UI). + JobStarted { + job_id: String, + title: String, + browse_url: String, + }, /// Tool requires user approval before execution. ApprovalNeeded { request_id: String, @@ -121,6 +127,19 @@ pub enum StatusUpdate { description: String, parameters: serde_json::Value, }, + /// Extension needs user authentication (token or OAuth). + AuthRequired { + extension_name: String, + instructions: Option, + auth_url: Option, + setup_url: Option, + }, + /// Extension authentication completed. + AuthCompleted { + extension_name: String, + success: bool, + message: String, + }, } /// Trait for message channels. diff --git a/src/channels/repl.rs b/src/channels/repl.rs index ef5db362..1dde2f47 100644 --- a/src/channels/repl.rs +++ b/src/channels/repl.rs @@ -33,21 +33,41 @@ use termimad::MadSkin; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; +use crate::agent::truncate_for_preview; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::error::ChannelError; +/// Max characters for tool result previews in the terminal. +const CLI_TOOL_RESULT_MAX: usize = 200; + +/// Max characters for thinking/status messages in the terminal. +const CLI_STATUS_MAX: usize = 200; + /// Slash commands available in the REPL. const SLASH_COMMANDS: &[&str] = &[ "/help", "/quit", "/exit", "/debug", + "/model", "/undo", "/redo", "/clear", "/compact", "/new", "/interrupt", + "/version", + "/tools", + "/ping", + "/job", + "/status", + "/cancel", + "/list", + "/heartbeat", + "/summarize", + "/suggest", + "/thread", + "/resume", ]; /// Rustyline helper for slash-command tab completion. @@ -248,7 +268,7 @@ impl Channel for ReplChannel { std::thread::spawn(move || { // Single message mode: send it and return if let Some(msg) = single_message { - let incoming = IncomingMessage::new("repl", "user", &msg); + let incoming = IncomingMessage::new("repl", "default", &msg); let _ = tx.blocking_send(incoming); return; } @@ -295,10 +315,11 @@ impl Channel for ReplChannel { continue; } - // Handle local REPL commands + // Handle local REPL commands (only commands that need + // immediate local handling stay here) match line.to_lowercase().as_str() { "/quit" | "/exit" => break, - "/help" | "/?" => { + "/help" => { print_help(); continue; } @@ -315,21 +336,21 @@ impl Channel for ReplChannel { _ => {} } - let msg = IncomingMessage::new("repl", "user", line); + let msg = IncomingMessage::new("repl", "default", line); if tx.blocking_send(msg).is_err() { break; } } Err(ReadlineError::Interrupted) => { // Ctrl+C: send /interrupt - let msg = IncomingMessage::new("repl", "user", "/interrupt"); + let msg = IncomingMessage::new("repl", "default", "/interrupt"); if tx.blocking_send(msg).is_err() { break; } } Err(ReadlineError::Eof) => { // Ctrl+D: send /quit so the agent loop runs graceful shutdown - let msg = IncomingMessage::new("repl", "user", "/quit"); + let msg = IncomingMessage::new("repl", "default", "/quit"); let _ = tx.blocking_send(msg); break; } @@ -386,7 +407,8 @@ impl Channel for ReplChannel { match status { StatusUpdate::Thinking(msg) => { - eprintln!(" \x1b[90m\u{25CB} {msg}\x1b[0m"); + let display = truncate_for_preview(&msg, CLI_STATUS_MAX); + eprintln!(" \x1b[90m\u{25CB} {display}\x1b[0m"); } StatusUpdate::ToolStarted { name } => { eprintln!(" \x1b[33m\u{25CB} {name}\x1b[0m"); @@ -399,7 +421,8 @@ impl Channel for ReplChannel { } } StatusUpdate::ToolResult { name: _, preview } => { - eprintln!(" \x1b[90m{preview}\x1b[0m"); + let display = truncate_for_preview(&preview, CLI_TOOL_RESULT_MAX); + eprintln!(" \x1b[90m{display}\x1b[0m"); } StatusUpdate::StreamChunk(chunk) => { // Print separator on the false-to-true transition @@ -413,9 +436,19 @@ impl Channel for ReplChannel { print!("{chunk}"); let _ = io::stdout().flush(); } + StatusUpdate::JobStarted { + job_id, + title, + browse_url, + } => { + eprintln!( + " \x1b[36m[job]\x1b[0m {title} \x1b[90m({job_id})\x1b[0m \x1b[4m{browse_url}\x1b[0m" + ); + } StatusUpdate::Status(msg) => { if debug || msg.contains("approval") || msg.contains("Approval") { - eprintln!(" \x1b[90m{msg}\x1b[0m"); + let display = truncate_for_preview(&msg, CLI_STATUS_MAX); + eprintln!(" \x1b[90m{display}\x1b[0m"); } } StatusUpdate::ApprovalNeeded { @@ -472,6 +505,33 @@ impl Channel for ReplChannel { eprintln!(" {bot_border}"); eprintln!(); } + StatusUpdate::AuthRequired { + extension_name, + instructions, + setup_url, + .. + } => { + eprintln!(); + eprintln!("\x1b[33m Authentication required for {extension_name}\x1b[0m"); + if let Some(ref instr) = instructions { + eprintln!(" {instr}"); + } + if let Some(ref url) = setup_url { + eprintln!(" \x1b[4m{url}\x1b[0m"); + } + eprintln!(); + } + StatusUpdate::AuthCompleted { + extension_name, + success, + message, + } => { + if success { + eprintln!("\x1b[32m {extension_name}: {message}\x1b[0m"); + } else { + eprintln!("\x1b[31m {extension_name}: {message}\x1b[0m"); + } + } } Ok(()) } diff --git a/src/channels/wasm/bundled.rs b/src/channels/wasm/bundled.rs index 9825b72b..1974be41 100644 --- a/src/channels/wasm/bundled.rs +++ b/src/channels/wasm/bundled.rs @@ -1,68 +1,125 @@ -//! Bundled WASM channels that can be installed locally. +//! Known WASM channels that can be installed from build artifacts. +//! +//! Instead of embedding WASM binaries in the host binary via include_bytes!, +//! channels are compiled separately and installed from their build output +//! directories during onboarding. +//! +//! Channel source layout: +//! channels-src// +//! target/wasm32-wasip2/release/_channel.wasm +//! .capabilities.json -use std::path::Path; +use std::path::{Path, PathBuf}; use tokio::fs; -#[derive(Clone, Copy)] -struct BundledChannel { - name: &'static str, - wasm: &'static [u8], - capabilities: &'static [u8], +/// Compile-time project root, used to locate channels-src/ in dev builds. +const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); + +/// Known channel names and their crate names (for locating build artifacts). +const KNOWN_CHANNELS: &[(&str, &str)] = &[ + ("telegram", "telegram_channel"), + ("slack", "slack_channel"), + ("whatsapp", "whatsapp_channel"), +]; + +/// Names of known channels that can be installed. +pub fn bundled_channel_names() -> Vec<&'static str> { + KNOWN_CHANNELS.iter().map(|(name, _)| *name).collect() } -/// Names of bundled channels shipped with IronClaw. -pub fn bundled_channel_names() -> &'static [&'static str] { - &["telegram"] +/// Resolve the channels source directory. +/// +/// Checks (in order): +/// 1. `IRONCLAW_CHANNELS_SRC` env var +/// 2. `/channels-src/` (dev builds) +fn channels_src_dir() -> PathBuf { + if let Ok(dir) = std::env::var("IRONCLAW_CHANNELS_SRC") { + return PathBuf::from(dir); + } + PathBuf::from(CARGO_MANIFEST_DIR).join("channels-src") } -/// Install a bundled channel into a channels directory. +/// Locate the build artifacts for a channel. +/// +/// Returns (wasm_path, capabilities_path) or an error if files are missing. +fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> { + let (_, crate_name) = KNOWN_CHANNELS + .iter() + .find(|(n, _)| *n == name) + .ok_or_else(|| format!("Unknown channel '{}'", name))?; + + let src_dir = channels_src_dir(); + let channel_dir = src_dir.join(name); + + let wasm_path = channel_dir + .join("target/wasm32-wasip2/release") + .join(format!("{}.wasm", crate_name)); + + let caps_path = channel_dir.join(format!("{}.capabilities.json", name)); + + if !wasm_path.exists() { + return Err(format!( + "Channel '{}' WASM not found at {}. Build it first:\n \ + cd {} && cargo build --target wasm32-wasip2 --release", + name, + wasm_path.display(), + channel_dir.display() + )); + } + + if !caps_path.exists() { + return Err(format!( + "Channel '{}' capabilities not found at {}", + name, + caps_path.display() + )); + } + + Ok((wasm_path, caps_path)) +} + +/// Install a channel from build artifacts into the channels directory. pub async fn install_bundled_channel( name: &str, target_dir: &Path, force: bool, ) -> Result<(), String> { - let channel = bundled_channel(name) - .ok_or_else(|| format!("Unknown bundled channel '{}'", name.to_lowercase()))?; + let (wasm_src, caps_src) = locate_channel_artifacts(name)?; fs::create_dir_all(target_dir) .await .map_err(|e| format!("Failed to create channels directory: {}", e))?; - let wasm_path = target_dir.join(format!("{}.wasm", channel.name)); - let caps_path = target_dir.join(format!("{}.capabilities.json", channel.name)); + let wasm_dst = target_dir.join(format!("{}.wasm", name)); + let caps_dst = target_dir.join(format!("{}.capabilities.json", name)); - let has_existing = wasm_path.exists() || caps_path.exists(); + let has_existing = wasm_dst.exists() || caps_dst.exists(); if has_existing && !force { return Err(format!( "Channel '{}' already exists at {}", - channel.name, + name, target_dir.display() )); } - fs::write(&wasm_path, channel.wasm) + fs::copy(&wasm_src, &wasm_dst) .await - .map_err(|e| format!("Failed to write {}: {}", wasm_path.display(), e))?; - fs::write(&caps_path, channel.capabilities) + .map_err(|e| format!("Failed to copy {}: {}", wasm_src.display(), e))?; + fs::copy(&caps_src, &caps_dst) .await - .map_err(|e| format!("Failed to write {}: {}", caps_path.display(), e))?; + .map_err(|e| format!("Failed to copy {}: {}", caps_src.display(), e))?; Ok(()) } -fn bundled_channel(name: &str) -> Option { - if name.eq_ignore_ascii_case("telegram") { - Some(BundledChannel { - name: "telegram", - wasm: include_bytes!("../../../channels-src/telegram/telegram.wasm"), - capabilities: include_bytes!( - "../../../channels-src/telegram/telegram.capabilities.json" - ), - }) - } else { - None - } +/// Check which known channels have build artifacts available. +pub fn available_channel_names() -> Vec<&'static str> { + KNOWN_CHANNELS + .iter() + .filter(|(name, _)| locate_channel_artifacts(name).is_ok()) + .map(|(name, _)| *name) + .collect() } #[cfg(test)] @@ -73,31 +130,35 @@ mod tests { use super::*; #[test] - fn test_bundled_channel_names_contains_telegram() { - assert!(bundled_channel_names().contains(&"telegram")); + fn test_known_channels_includes_all_three() { + let names = bundled_channel_names(); + assert!(names.contains(&"telegram")); + assert!(names.contains(&"slack")); + assert!(names.contains(&"whatsapp")); + } + + #[test] + fn test_channels_src_dir_default() { + let dir = channels_src_dir(); + assert!(dir.ends_with("channels-src")); + } + + #[test] + fn test_locate_unknown_channel_errors() { + assert!(locate_channel_artifacts("nonexistent").is_err()); } #[tokio::test] - async fn test_install_bundled_channel_writes_files() { - let dir = tempdir().unwrap(); - - install_bundled_channel("telegram", dir.path(), false) - .await - .unwrap(); - - assert!(dir.path().join("telegram.wasm").exists()); - assert!(dir.path().join("telegram.capabilities.json").exists()); - } - - #[tokio::test] - async fn test_install_bundled_channel_refuses_overwrite_without_force() { + async fn test_install_refuses_overwrite_without_force() { let dir = tempdir().unwrap(); let wasm_path = dir.path().join("telegram.wasm"); fs::write(&wasm_path, b"custom").await.unwrap(); let result = install_bundled_channel("telegram", dir.path(), false).await; + // Either fails because artifacts missing OR because file exists assert!(result.is_err()); + // Original file should be untouched let existing = fs::read(&wasm_path).await.unwrap(); assert_eq!(existing, b"custom"); } diff --git a/src/channels/wasm/loader.rs b/src/channels/wasm/loader.rs index 00dfcd80..3f7fdb63 100644 --- a/src/channels/wasm/loader.rs +++ b/src/channels/wasm/loader.rs @@ -16,16 +16,21 @@ use crate::channels::wasm::error::WasmChannelError; use crate::channels::wasm::runtime::WasmChannelRuntime; use crate::channels::wasm::schema::ChannelCapabilitiesFile; use crate::channels::wasm::wrapper::WasmChannel; +use crate::pairing::PairingStore; /// Loads WASM channels from the filesystem. pub struct WasmChannelLoader { runtime: Arc, + pairing_store: Arc, } impl WasmChannelLoader { - /// Create a new loader with the given runtime. - pub fn new(runtime: Arc) -> Self { - Self { runtime } + /// Create a new loader with the given runtime and pairing store. + pub fn new(runtime: Arc, pairing_store: Arc) -> Self { + Self { + runtime, + pairing_store, + } } /// Load a single WASM channel from a file pair. @@ -114,7 +119,13 @@ impl WasmChannelLoader { .await?; // Create the channel - let channel = WasmChannel::new(self.runtime.clone(), prepared, capabilities, config_json); + let channel = WasmChannel::new( + self.runtime.clone(), + prepared, + capabilities, + config_json, + self.pairing_store.clone(), + ); tracing::info!( name = name, @@ -352,6 +363,7 @@ mod tests { use crate::channels::wasm::loader::{WasmChannelLoader, discover_channels}; use crate::channels::wasm::runtime::{WasmChannelRuntime, WasmChannelRuntimeConfig}; + use crate::pairing::PairingStore; use std::sync::Arc; #[tokio::test] @@ -408,7 +420,7 @@ mod tests { async fn test_loader_invalid_name() { let config = WasmChannelRuntimeConfig::for_testing(); let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap()); - let loader = WasmChannelLoader::new(runtime); + let loader = WasmChannelLoader::new(runtime, Arc::new(PairingStore::new())); let dir = TempDir::new().unwrap(); let wasm_path = dir.path().join("test.wasm"); diff --git a/src/channels/wasm/mod.rs b/src/channels/wasm/mod.rs index 9f7b7c37..17ac7726 100644 --- a/src/channels/wasm/mod.rs +++ b/src/channels/wasm/mod.rs @@ -89,7 +89,7 @@ mod schema; mod wrapper; // Core types -pub use bundled::{bundled_channel_names, install_bundled_channel}; +pub use bundled::{available_channel_names, bundled_channel_names, install_bundled_channel}; pub use capabilities::{ChannelCapabilities, EmitRateLimitConfig, HttpEndpointConfig, PollConfig}; pub use error::WasmChannelError; pub use host::{ChannelEmitRateLimiter, ChannelHostState, EmittedMessage}; diff --git a/src/channels/wasm/router.rs b/src/channels/wasm/router.rs index 0bd3182f..cbf8b7b8 100644 --- a/src/channels/wasm/router.rs +++ b/src/channels/wasm/router.rs @@ -478,6 +478,7 @@ mod tests { PreparedChannelModule, WasmChannelRuntime, WasmChannelRuntimeConfig, }; use crate::channels::wasm::wrapper::WasmChannel; + use crate::pairing::PairingStore; use crate::tools::wasm::ResourceLimits; fn create_test_channel(name: &str) -> Arc { @@ -499,6 +500,7 @@ mod tests { prepared, capabilities, "{}".to_string(), + Arc::new(PairingStore::new()), )) } diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 14a33a24..212334a6 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -48,6 +48,7 @@ use crate::channels::wasm::runtime::{PreparedChannelModule, WasmChannelRuntime}; use crate::channels::wasm::schema::ChannelConfig; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::error::ChannelError; +use crate::pairing::PairingStore; use crate::safety::LeakDetector; use crate::tools::wasm::LogLevel; use crate::tools::wasm::WasmResourceLimiter; @@ -73,6 +74,11 @@ struct ChannelStoreData { /// Injected credentials for URL substitution (e.g., bot tokens). /// Keys are placeholder names like "TELEGRAM_BOT_TOKEN". credentials: HashMap, + /// Pairing store for DM pairing (guest access control). + pairing_store: Arc, + /// Dedicated tokio runtime for HTTP requests, lazily initialized. + /// Reused across multiple `http_request` calls within one execution. + http_runtime: Option, } impl ChannelStoreData { @@ -81,6 +87,7 @@ impl ChannelStoreData { channel_name: &str, capabilities: ChannelCapabilities, credentials: HashMap, + pairing_store: Arc, ) -> Self { // Create a minimal WASI context (no filesystem, no env vars for security) let wasi = WasiCtxBuilder::new().build(); @@ -91,6 +98,8 @@ impl ChannelStoreData { wasi, table: ResourceTable::new(), credentials, + pairing_store, + http_runtime: None, } } @@ -129,18 +138,34 @@ impl ChannelStoreData { if result.contains('{') && result.contains('}') { // Only warn if it looks like an unresolved placeholder (not JSON braces) let brace_pattern = regex::Regex::new(r"\{[A-Z_]+\}").ok(); - if let Some(re) = brace_pattern { - if re.is_match(&result) { - tracing::warn!( - context = %context, - "String may contain unresolved credential placeholders" - ); - } + if let Some(re) = brace_pattern + && re.is_match(&result) + { + tracing::warn!( + context = %context, + "String may contain unresolved credential placeholders" + ); } } result } + + /// Replace injected credential values with `[REDACTED]` in text. + /// + /// Prevents credentials from leaking through error messages, logs, or + /// return values to WASM. reqwest::Error includes the full URL in its + /// Display output, so any error from an injected-URL request will + /// contain the raw credential unless we scrub it. + fn redact_credentials(&self, text: &str) -> String { + let mut result = text.to_string(); + for (name, value) in &self.credentials { + if !value.is_empty() { + result = result.replace(value, &format!("[REDACTED:{}]", name)); + } + } + result + } } // Implement WasiView to provide WASI context and resource table @@ -187,6 +212,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { url: String, headers_json: String, body: Option>, + timeout_ms: Option, ) -> Result { tracing::info!( method = %method, @@ -251,10 +277,35 @@ impl near::agent::channel_host::Host for ChannelStoreData { .scan_http_request(&url, &header_vec, body.as_deref()) .map_err(|e| format!("Potential secret leak blocked: {}", e))?; - // Make the HTTP request using blocking I/O - // We're already in a spawn_blocking context, so we can use block_on - let result = tokio::runtime::Handle::current().block_on(async { - let client = reqwest::Client::new(); + // Get the max response size from capabilities (default 10MB). + let max_response_bytes = self + .host_state + .capabilities() + .tool_capabilities + .http + .as_ref() + .map(|h| h.max_response_bytes) + .unwrap_or(10 * 1024 * 1024); + + // Make the HTTP request using a dedicated single-threaded runtime. + // We're inside spawn_blocking, so we can't rely on the main runtime's + // I/O driver (it may be busy with WASM compilation or other startup work). + // A dedicated runtime gives us our own I/O driver and avoids contention. + // The runtime is lazily created and reused across calls within one execution. + if self.http_runtime.is_none() { + self.http_runtime = Some( + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("Failed to create HTTP runtime: {e}"))?, + ); + } + let rt = self.http_runtime.as_ref().expect("just initialized"); + let result = rt.block_on(async { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .map_err(|e| format!("Failed to build HTTP client: {e}"))?; let mut request = match method.to_uppercase().as_str() { "GET" => client.get(&url), @@ -276,12 +327,21 @@ impl near::agent::channel_host::Host for ChannelStoreData { request = request.body(body_bytes); } - // Send request with timeout - let response = request - .timeout(std::time::Duration::from_secs(30)) - .send() - .await - .map_err(|e| format!("HTTP request failed: {}", e))?; + // Send request with caller-specified timeout (default 30s, max 5min). + let timeout_ms = timeout_ms.unwrap_or(30_000).min(300_000) as u64; + let timeout = std::time::Duration::from_millis(timeout_ms); + let response = request.timeout(timeout).send().await.map_err(|e| { + // Walk the full error chain so we get the actual root cause + // (DNS, TLS, connection refused, etc.) instead of just + // "error sending request for url (...)". + let mut chain = format!("HTTP request failed: {}", e); + let mut source = std::error::Error::source(&e); + while let Some(cause) = source { + chain.push_str(&format!(" -> {}", cause)); + source = cause.source(); + } + chain + })?; let status = response.status().as_u16(); let response_headers: std::collections::HashMap = response @@ -294,11 +354,29 @@ impl near::agent::channel_host::Host for ChannelStoreData { }) .collect(); let headers_json = serde_json::to_string(&response_headers).unwrap_or_default(); + + // Enforce max response body size to prevent memory exhaustion. + let max_response = max_response_bytes; + if let Some(cl) = response.content_length() + && cl as usize > max_response + { + return Err(format!( + "Response body too large: {} bytes exceeds limit of {} bytes", + cl, max_response + )); + } let body = response .bytes() .await - .map_err(|e| format!("Failed to read response body: {}", e))? - .to_vec(); + .map_err(|e| format!("Failed to read response body: {}", e))?; + if body.len() > max_response { + return Err(format!( + "Response body too large: {} bytes exceeds limit of {} bytes", + body.len(), + max_response + )); + } + let body = body.to_vec(); tracing::info!( status = status, @@ -330,6 +408,11 @@ impl near::agent::channel_host::Host for ChannelStoreData { }) }); + // Scrub credential values from error messages before logging or returning + // to WASM. reqwest::Error includes the full URL (with injected credentials) + // in its Display output. + let result = result.map_err(|e| self.redact_credentials(&e)); + match &result { Ok(resp) => { tracing::info!(status = resp.status, "http_request completed successfully"); @@ -372,6 +455,43 @@ impl near::agent::channel_host::Host for ChannelStoreData { } } } + + fn pairing_upsert_request( + &mut self, + channel: String, + id: String, + meta_json: String, + ) -> Result { + let meta = if meta_json.is_empty() { + None + } else { + serde_json::from_str(&meta_json).ok() + }; + match self.pairing_store.upsert_request(&channel, &id, meta) { + Ok(r) => Ok(near::agent::channel_host::PairingUpsertResult { + code: r.code, + created: r.created, + }), + Err(e) => Err(e.to_string()), + } + } + + fn pairing_is_allowed( + &mut self, + channel: String, + id: String, + username: Option, + ) -> Result { + self.pairing_store + .is_sender_allowed(&channel, &id, username.as_deref()) + .map_err(|e| e.to_string()) + } + + fn pairing_read_allow_from(&mut self, channel: String) -> Result, String> { + self.pairing_store + .read_allow_from(&channel) + .map_err(|e| e.to_string()) + } } /// A WASM-based channel implementing the Channel trait. @@ -424,6 +544,9 @@ pub struct WasmChannel { /// Background task that repeats typing indicators every 4 seconds. /// Telegram's "typing..." indicator expires after ~5s, so we refresh it. typing_task: RwLock>>, + + /// Pairing store for DM pairing (guest access control). + pairing_store: Arc, } impl WasmChannel { @@ -433,6 +556,7 @@ impl WasmChannel { prepared: Arc, capabilities: ChannelCapabilities, config_json: String, + pairing_store: Arc, ) -> Self { let name = prepared.name.clone(); let rate_limiter = ChannelEmitRateLimiter::new(capabilities.emit_rate_limit.clone()); @@ -452,6 +576,7 @@ impl WasmChannel { endpoints: RwLock::new(Vec::new()), credentials: Arc::new(RwLock::new(HashMap::new())), typing_task: RwLock::new(None), + pairing_store, } } @@ -533,6 +658,7 @@ impl WasmChannel { prepared: &PreparedChannelModule, capabilities: &ChannelCapabilities, credentials: HashMap, + pairing_store: Arc, ) -> Result, WasmChannelError> { let engine = runtime.engine(); let limits = &prepared.limits; @@ -543,6 +669,7 @@ impl WasmChannel { &prepared.name, capabilities.clone(), credentials, + pairing_store, ); let mut store = Store::new(engine, store_data); @@ -643,12 +770,18 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let pairing_store = self.pairing_store.clone(); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Call on_start using the generated typed interface @@ -681,7 +814,21 @@ impl WasmChannel { .await; match result { - Ok(Ok((config, _host_state))) => { + Ok(Ok((config, mut host_state))) => { + // Surface WASM guest logs (errors/warnings from webhook setup, etc.) + for entry in host_state.take_logs() { + match entry.level { + crate::tools::wasm::LogLevel::Error => { + tracing::error!(channel = %self.name, "{}", entry.message); + } + crate::tools::wasm::LogLevel::Warn => { + tracing::warn!(channel = %self.name, "{}", entry.message); + } + _ => { + tracing::debug!(channel = %self.name, "{}", entry.message); + } + } + } tracing::info!( channel = %self.name, display_name = %config.display_name, @@ -753,6 +900,7 @@ impl WasmChannel { let capabilities = self.capabilities.clone(); let timeout = self.runtime.config().callback_timeout; let credentials = self.get_credentials().await; + let pairing_store = self.pairing_store.clone(); // Prepare request data let method = method.to_string(); @@ -766,8 +914,13 @@ impl WasmChannel { // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Build the WIT request type @@ -840,12 +993,18 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let pairing_store = self.pairing_store.clone(); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Call on_poll using the generated typed interface @@ -929,6 +1088,7 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let pairing_store = self.pairing_store.clone(); // Prepare response data let message_id_str = message_id.to_string(); @@ -942,8 +1102,13 @@ impl WasmChannel { let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { tracing::info!("Creating WASM store for on_respond"); - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + pairing_store, + )?; tracing::info!("Instantiating WASM component for on_respond"); let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -1036,13 +1201,19 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let pairing_store = self.pairing_store.clone(); let wit_update = status_to_wit(status, metadata); let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; let channel_iface = instance.near_agent_channel(); @@ -1080,12 +1251,14 @@ impl WasmChannel { /// /// Static method for use by the background typing repeat task (which /// doesn't have access to `&self`). + #[allow(clippy::too_many_arguments)] async fn execute_status( channel_name: &str, runtime: &Arc, prepared: &Arc, capabilities: &ChannelCapabilities, credentials: &RwLock>, + pairing_store: Arc, timeout: Duration, wit_update: wit_channel::StatusUpdate, ) -> Result<(), WasmChannelError> { @@ -1101,8 +1274,13 @@ impl WasmChannel { let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials_snapshot)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials_snapshot, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; let channel_iface = instance.near_agent_channel(); @@ -1170,6 +1348,7 @@ impl WasmChannel { let prepared = Arc::clone(&self.prepared); let capabilities = self.capabilities.clone(); let credentials = self.credentials.clone(); + let pairing_store = self.pairing_store.clone(); let callback_timeout = self.runtime.config().callback_timeout; let wit_update = status_to_wit(&status, metadata); @@ -1189,6 +1368,7 @@ impl WasmChannel { &prepared, &capabilities, &credentials, + pairing_store.clone(), callback_timeout, wit_update_clone, ) @@ -1319,6 +1499,7 @@ impl WasmChannel { let message_tx = self.message_tx.clone(); let rate_limiter = self.rate_limiter.clone(); let credentials = self.credentials.clone(); + let pairing_store = self.pairing_store.clone(); let callback_timeout = self.runtime.config().callback_timeout; tokio::spawn(async move { @@ -1340,14 +1521,15 @@ impl WasmChannel { &prepared, &capabilities, &credentials, + pairing_store.clone(), callback_timeout, ).await; match result { Ok(emitted_messages) => { // Process any emitted messages - if !emitted_messages.is_empty() { - if let Err(e) = Self::dispatch_emitted_messages( + if !emitted_messages.is_empty() + && let Err(e) = Self::dispatch_emitted_messages( &channel_name, emitted_messages, &message_tx, @@ -1359,7 +1541,6 @@ impl WasmChannel { "Failed to dispatch emitted messages from poll" ); } - } } Err(e) => { tracing::warn!( @@ -1391,6 +1572,7 @@ impl WasmChannel { prepared: &Arc, capabilities: &ChannelCapabilities, credentials: &RwLock>, + pairing_store: Arc, timeout: Duration, ) -> Result, WasmChannelError> { // Skip if no WASM bytes (testing mode) @@ -1411,8 +1593,13 @@ impl WasmChannel { // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { - let mut store = - Self::create_store(&runtime, &prepared, &capabilities, credentials_snapshot)?; + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials_snapshot, + pairing_store, + )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; // Call on_poll using the generated typed interface @@ -1583,22 +1770,22 @@ impl Channel for WasmChannel { *self.endpoints.write().await = endpoints; // Start polling if configured - if let Some(poll_config) = &config.poll { - if poll_config.enabled { - let interval = self - .capabilities - .validate_poll_interval(poll_config.interval_ms) - .map_err(|e| ChannelError::StartupFailed { - name: self.name.clone(), - reason: e, - })?; + if let Some(poll_config) = &config.poll + && poll_config.enabled + { + let interval = self + .capabilities + .validate_poll_interval(poll_config.interval_ms) + .map_err(|e| ChannelError::StartupFailed { + name: self.name.clone(), + reason: e, + })?; - // Create shutdown channel for polling and store the sender to keep it alive - let (poll_shutdown_tx, poll_shutdown_rx) = oneshot::channel(); - *self.poll_shutdown_tx.write().await = Some(poll_shutdown_tx); + // Create shutdown channel for polling and store the sender to keep it alive + let (poll_shutdown_tx, poll_shutdown_rx) = oneshot::channel(); + *self.poll_shutdown_tx.write().await = Some(poll_shutdown_tx); - self.start_polling(Duration::from_millis(interval as u64), poll_shutdown_rx); - } + self.start_polling(Duration::from_millis(interval as u64), poll_shutdown_rx); } tracing::info!( @@ -1858,6 +2045,29 @@ fn status_to_wit(status: &StatusUpdate, metadata: &serde_json::Value) -> wit_cha message: format!("Approval needed: {} - {}", tool_name, description), metadata_json, }, + StatusUpdate::JobStarted { job_id, title, .. } => wit_channel::StatusUpdate { + status: wit_channel::StatusType::Thinking, + message: format!("Job started: {} ({})", title, job_id), + metadata_json, + }, + StatusUpdate::AuthRequired { extension_name, .. } => wit_channel::StatusUpdate { + status: wit_channel::StatusType::Thinking, + message: format!("Auth required: {}", extension_name), + metadata_json, + }, + StatusUpdate::AuthCompleted { + extension_name, + success, + .. + } => wit_channel::StatusUpdate { + status: wit_channel::StatusType::Thinking, + message: format!( + "Auth {}: {}", + if *success { "completed" } else { "failed" }, + extension_name + ), + metadata_json, + }, } } @@ -1929,6 +2139,7 @@ mod tests { PreparedChannelModule, WasmChannelRuntime, WasmChannelRuntimeConfig, }; use crate::channels::wasm::wrapper::{HttpResponse, WasmChannel}; + use crate::pairing::PairingStore; use crate::tools::wasm::ResourceLimits; fn create_test_channel() -> WasmChannel { @@ -1944,7 +2155,13 @@ mod tests { let capabilities = ChannelCapabilities::for_channel("test").with_path("/webhook/test"); - WasmChannel::new(runtime, prepared, capabilities, "{}".to_string()) + WasmChannel::new( + runtime, + prepared, + capabilities, + "{}".to_string(), + Arc::new(PairingStore::new()), + ) } #[test] @@ -2019,6 +2236,7 @@ mod tests { &prepared, &capabilities, &credentials, + Arc::new(PairingStore::new()), timeout, ) .await; @@ -2112,7 +2330,13 @@ mod tests { .with_path("/webhook/poll") .with_polling(1000); - let channel = WasmChannel::new(runtime, prepared, capabilities, "{}".to_string()); + let channel = WasmChannel::new( + runtime, + prepared, + capabilities, + "{}".to_string(), + Arc::new(PairingStore::new()), + ); // Start the channel let _stream = channel.start().await.expect("Channel should start"); @@ -2350,4 +2574,126 @@ mod tests { assert_eq!(cloned.message, "hello"); assert_eq!(cloned.metadata_json, "{\"a\":1}"); } + + #[test] + fn test_redact_credentials_replaces_values() { + use super::ChannelStoreData; + + let mut creds = std::collections::HashMap::new(); + creds.insert( + "TELEGRAM_BOT_TOKEN".to_string(), + "8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis".to_string(), + ); + creds.insert("OTHER_SECRET".to_string(), "s3cret".to_string()); + + let store = ChannelStoreData::new( + 1024 * 1024, + "test", + ChannelCapabilities::default(), + creds, + Arc::new(PairingStore::new()), + ); + + let error = "HTTP request failed: error sending request for url \ + (https://api.telegram.org/bot8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis/getUpdates)"; + + let redacted = store.redact_credentials(error); + + assert!( + !redacted.contains("8218490433:AAEZeUxwqZ5OO3mOCXv7fKvpdhDgsmBBNis"), + "credential value should be redacted" + ); + assert!( + redacted.contains("[REDACTED:TELEGRAM_BOT_TOKEN]"), + "redacted text should contain placeholder name" + ); + assert!( + !redacted.contains("s3cret"), + "other credentials should also be redacted" + ); + } + + #[test] + fn test_redact_credentials_no_op_without_credentials() { + use super::ChannelStoreData; + + let store = ChannelStoreData::new( + 1024 * 1024, + "test", + ChannelCapabilities::default(), + std::collections::HashMap::new(), + Arc::new(PairingStore::new()), + ); + + let input = "some error message"; + assert_eq!(store.redact_credentials(input), input); + } + + #[test] + fn test_redact_credentials_skips_empty_values() { + use super::ChannelStoreData; + + let mut creds = std::collections::HashMap::new(); + creds.insert("EMPTY_TOKEN".to_string(), String::new()); + + let store = ChannelStoreData::new( + 1024 * 1024, + "test", + ChannelCapabilities::default(), + creds, + Arc::new(PairingStore::new()), + ); + + let input = "should not match anything"; + assert_eq!(store.redact_credentials(input), input); + } + + /// Verify that WASM HTTP host functions work using a dedicated + /// current-thread runtime inside spawn_blocking. + #[tokio::test] + async fn test_dedicated_runtime_inside_spawn_blocking() { + let result = tokio::task::spawn_blocking(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime"); + rt.block_on(async { 42 }) + }) + .await + .expect("spawn_blocking panicked"); + assert_eq!(result, 42); + } + + /// Verify a real HTTP request works using the dedicated-runtime pattern. + /// This catches DNS, TLS, and I/O driver issues that trivial tests miss. + #[tokio::test] + #[ignore] // requires network + async fn test_dedicated_runtime_real_http() { + let result = tokio::task::spawn_blocking(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime"); + rt.block_on(async { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .expect("failed to build client"); + let resp = client + .get("https://api.telegram.org/bot000/getMe") + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + match resp { + Ok(r) => r.status().as_u16(), + Err(e) if e.is_timeout() => panic!("request timed out: {e}"), + Err(e) => panic!("unexpected error: {e}"), + } + }) + }) + .await + .expect("spawn_blocking panicked"); + // 404 because "000" is not a valid bot token + assert_eq!(result, 404); + } } diff --git a/src/channels/web/auth.rs b/src/channels/web/auth.rs index 219528eb..23d1ddfc 100644 --- a/src/channels/web/auth.rs +++ b/src/channels/web/auth.rs @@ -6,6 +6,7 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; +use subtle::ConstantTimeEq; /// Shared auth state injected via axum middleware state. #[derive(Clone)] @@ -23,24 +24,22 @@ pub async fn auth_middleware( request: Request, next: Next, ) -> Response { - // Try Authorization header first - if let Some(auth_header) = headers.get("authorization") { - if let Ok(value) = auth_header.to_str() { - if let Some(token) = value.strip_prefix("Bearer ") { - if token == auth.token { - return next.run(request).await; - } - } - } + // Try Authorization header first (constant-time comparison) + if let Some(auth_header) = headers.get("authorization") + && let Ok(value) = auth_header.to_str() + && let Some(token) = value.strip_prefix("Bearer ") + && bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) + { + return next.run(request).await; } - // Fall back to query parameter (for SSE EventSource) + // Fall back to query parameter for SSE EventSource (constant-time comparison) if let Some(query) = request.uri().query() { for pair in query.split('&') { - if let Some(token) = pair.strip_prefix("token=") { - if token == auth.token { - return next.run(request).await; - } + if let Some(token) = pair.strip_prefix("token=") + && bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) + { + return next.run(request).await; } } } diff --git a/src/channels/web/log_layer.rs b/src/channels/web/log_layer.rs index 02a8e4ed..3a55b994 100644 --- a/src/channels/web/log_layer.rs +++ b/src/channels/web/log_layer.rs @@ -24,6 +24,8 @@ use tokio::sync::broadcast; use tracing::field::{Field, Visit}; use tracing_subscriber::Layer; +use crate::safety::LeakDetector; + /// Maximum number of recent log entries kept for late-joining SSE subscribers. const HISTORY_CAP: usize = 500; @@ -46,6 +48,8 @@ pub struct LogEntry { pub struct LogBroadcaster { tx: broadcast::Sender, recent: Mutex>, + /// Scrubs secrets from log messages before broadcasting to SSE clients. + leak_detector: LeakDetector, } impl LogBroadcaster { @@ -54,10 +58,19 @@ impl LogBroadcaster { Self { tx, recent: Mutex::new(VecDeque::with_capacity(HISTORY_CAP)), + leak_detector: LeakDetector::new(), } } - pub fn send(&self, entry: LogEntry) { + pub fn send(&self, mut entry: LogEntry) { + // Scrub secrets from the message before it reaches any subscriber. + // This is defense-in-depth: even if code elsewhere accidentally logs + // a secret, it won't be broadcast to SSE clients. + entry.message = self + .leak_detector + .scan_and_clean(&entry.message) + .unwrap_or_else(|_| "[log message redacted: contained blocked secret]".to_string()); + // Stash in ring buffer (for late joiners) if let Ok(mut buf) = self.recent.lock() { if buf.len() >= HISTORY_CAP { @@ -145,6 +158,9 @@ impl Visit for MessageVisitor { /// /// Only forwards DEBUG and above. Attach to the tracing subscriber /// alongside the existing fmt layer. +/// +/// Log messages are scrubbed through `LeakDetector` in `LogBroadcaster::send()` +/// (the single funnel point for all log output, including late-joiner history). pub struct WebLogLayer { broadcaster: Arc, } @@ -178,6 +194,7 @@ impl Layer for WebLogLayer { timestamp: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true), }; + // LeakDetector scrubbing happens inside broadcaster.send() self.broadcaster.send(entry); } } @@ -313,4 +330,29 @@ mod tests { let v = MessageVisitor::new(); assert_eq!(v.finish(), ""); } + + #[test] + fn test_broadcaster_has_leak_detector() { + let broadcaster = LogBroadcaster::new(); + // Verify the leak detector is initialized with default patterns + assert!(broadcaster.leak_detector.pattern_count() > 0); + } + + #[test] + fn test_leak_detector_scrubs_api_key_in_log() { + let detector = crate::safety::LeakDetector::new(); + let msg = "Connecting with token sk-proj-test1234567890abcdefghij"; + let result = detector.scan_and_clean(msg); + // Should be blocked (OpenAI key pattern) + assert!(result.is_err()); + } + + #[test] + fn test_leak_detector_passes_clean_log() { + let detector = crate::safety::LeakDetector::new(); + let msg = "Request completed status=200 url=https://api.example.com/data"; + let result = detector.scan_and_clean(msg); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), msg); + } } diff --git a/src/channels/web/mod.rs b/src/channels/web/mod.rs index 46590402..356eda2c 100644 --- a/src/channels/web/mod.rs +++ b/src/channels/web/mod.rs @@ -10,12 +10,13 @@ //! ◄── GET /api/chat/events ── SSE stream //! ─── GET /api/chat/ws ─────► WebSocket (bidirectional) //! ─── GET /api/memory/* ────► Workspace -//! ─── GET /api/jobs/* ──────► ContextManager +//! ─── GET /api/jobs/* ──────► Database //! ◄── GET / ───────────────── Static HTML/CSS/JS //! ``` pub mod auth; pub mod log_layer; +pub mod openai_compat; pub mod server; pub mod sse; pub mod types; @@ -31,9 +32,10 @@ use tokio_stream::wrappers::ReceiverStream; use crate::agent::SessionManager; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::config::GatewayConfig; -use crate::context::ContextManager; +use crate::db::Database; use crate::error::ChannelError; use crate::extensions::ExtensionManager; +use crate::orchestrator::job_manager::ContainerJobManager; use crate::tools::ToolRegistry; use crate::workspace::Workspace; @@ -70,14 +72,18 @@ impl GatewayChannel { msg_tx: tokio::sync::RwLock::new(None), sse: SseManager::new(), workspace: None, - context_manager: None, session_manager: None, log_broadcaster: None, extension_manager: None, tool_registry: None, + store: None, + job_manager: None, + prompt_queue: None, user_id: config.user_id.clone(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: Some(Arc::new(ws::WsConnectionTracker::new())), + llm_provider: None, + chat_rate_limiter: server::RateLimiter::new(30, 60), }); Self { @@ -93,14 +99,18 @@ impl GatewayChannel { msg_tx: tokio::sync::RwLock::new(None), sse: SseManager::new(), workspace: self.state.workspace.clone(), - context_manager: self.state.context_manager.clone(), session_manager: self.state.session_manager.clone(), log_broadcaster: self.state.log_broadcaster.clone(), extension_manager: self.state.extension_manager.clone(), tool_registry: self.state.tool_registry.clone(), + store: self.state.store.clone(), + job_manager: self.state.job_manager.clone(), + prompt_queue: self.state.prompt_queue.clone(), user_id: self.state.user_id.clone(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: self.state.ws_tracker.clone(), + llm_provider: self.state.llm_provider.clone(), + chat_rate_limiter: server::RateLimiter::new(30, 60), }; mutate(&mut new_state); self.state = Arc::new(new_state); @@ -112,12 +122,6 @@ impl GatewayChannel { self } - /// Inject the context manager for the jobs API. - pub fn with_context_manager(mut self, cm: Arc) -> Self { - self.rebuild_state(|s| s.context_manager = Some(cm)); - self - } - /// Inject the session manager for thread/session info. pub fn with_session_manager(mut self, sm: Arc) -> Self { self.rebuild_state(|s| s.session_manager = Some(sm)); @@ -142,6 +146,40 @@ impl GatewayChannel { self } + /// Inject the database store for sandbox job persistence. + pub fn with_store(mut self, store: Arc) -> Self { + self.rebuild_state(|s| s.store = Some(store)); + self + } + + /// Inject the container job manager for sandbox operations. + pub fn with_job_manager(mut self, jm: Arc) -> Self { + self.rebuild_state(|s| s.job_manager = Some(jm)); + self + } + + /// Inject the prompt queue for Claude Code follow-up prompts. + pub fn with_prompt_queue( + mut self, + pq: Arc< + tokio::sync::Mutex< + std::collections::HashMap< + uuid::Uuid, + std::collections::VecDeque, + >, + >, + >, + ) -> Self { + self.rebuild_state(|s| s.prompt_queue = Some(pq)); + self + } + + /// Inject the LLM provider for OpenAI-compatible API proxy. + pub fn with_llm_provider(mut self, llm: Arc) -> Self { + self.rebuild_state(|s| s.llm_provider = Some(llm)); + self + } + /// Get the auth token (for printing to console on startup). pub fn auth_token(&self) -> &str { &self.auth_token @@ -173,11 +211,7 @@ impl Channel for GatewayChannel { ), })?; - let bound_addr = - server::start_server(addr, self.state.clone(), self.auth_token.clone()).await?; - - tracing::info!("Web gateway listening on http://{}", bound_addr); - tracing::info!("Auth token: {}", self.auth_token); + server::start_server(addr, self.state.clone(), self.auth_token.clone()).await?; Ok(Box::pin(ReceiverStream::new(rx))) } @@ -200,17 +234,48 @@ impl Channel for GatewayChannel { async fn send_status( &self, status: StatusUpdate, - _metadata: &serde_json::Value, + metadata: &serde_json::Value, ) -> Result<(), ChannelError> { + let thread_id = metadata + .get("thread_id") + .and_then(|v| v.as_str()) + .map(String::from); let event = match status { - StatusUpdate::Thinking(msg) => SseEvent::Thinking { message: msg }, - StatusUpdate::ToolStarted { name } => SseEvent::ToolStarted { name }, - StatusUpdate::ToolCompleted { name, success } => { - SseEvent::ToolCompleted { name, success } - } - StatusUpdate::ToolResult { name, preview } => SseEvent::ToolResult { name, preview }, - StatusUpdate::StreamChunk(content) => SseEvent::StreamChunk { content }, - StatusUpdate::Status(msg) => SseEvent::Status { message: msg }, + StatusUpdate::Thinking(msg) => SseEvent::Thinking { + message: msg, + thread_id: thread_id.clone(), + }, + StatusUpdate::ToolStarted { name } => SseEvent::ToolStarted { + name, + thread_id: thread_id.clone(), + }, + StatusUpdate::ToolCompleted { name, success } => SseEvent::ToolCompleted { + name, + success, + thread_id: thread_id.clone(), + }, + StatusUpdate::ToolResult { name, preview } => SseEvent::ToolResult { + name, + preview, + thread_id: thread_id.clone(), + }, + StatusUpdate::StreamChunk(content) => SseEvent::StreamChunk { + content, + thread_id: thread_id.clone(), + }, + StatusUpdate::Status(msg) => SseEvent::Status { + message: msg, + thread_id: thread_id.clone(), + }, + StatusUpdate::JobStarted { + job_id, + title, + browse_url, + } => SseEvent::JobStarted { + job_id, + title, + browse_url, + }, StatusUpdate::ApprovalNeeded { request_id, tool_name, @@ -223,6 +288,26 @@ impl Channel for GatewayChannel { parameters: serde_json::to_string_pretty(¶meters) .unwrap_or_else(|_| parameters.to_string()), }, + StatusUpdate::AuthRequired { + extension_name, + instructions, + auth_url, + setup_url, + } => SseEvent::AuthRequired { + extension_name, + instructions, + auth_url, + setup_url, + }, + StatusUpdate::AuthCompleted { + extension_name, + success, + message, + } => SseEvent::AuthCompleted { + extension_name, + success, + message, + }, }; self.state.sse.broadcast(event); diff --git a/src/channels/web/openai_compat.rs b/src/channels/web/openai_compat.rs new file mode 100644 index 00000000..b2dfa007 --- /dev/null +++ b/src/channels/web/openai_compat.rs @@ -0,0 +1,1094 @@ +//! OpenAI-compatible HTTP API (`/v1/chat/completions`, `/v1/models`). +//! +//! This module provides a direct LLM proxy through the web gateway so any +//! standard OpenAI client library can use IronClaw as a backend by simply +//! changing the `base_url`. + +use std::sync::Arc; + +use axum::{ + Json, + extract::State, + http::{HeaderValue, StatusCode}, + response::{ + IntoResponse, Response, + sse::{Event, KeepAlive, Sse}, + }, +}; +use serde::{Deserialize, Serialize}; + +use crate::llm::{ + ChatMessage, CompletionRequest, FinishReason, Role, ToolCall, ToolCompletionRequest, + ToolDefinition, +}; + +use super::server::GatewayState; + +// --------------------------------------------------------------------------- +// OpenAI request types +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +pub struct OpenAiChatRequest { + pub model: String, + pub messages: Vec, + #[serde(default)] + pub temperature: Option, + #[serde(default)] + pub max_tokens: Option, + #[serde(default)] + pub stream: Option, + #[serde(default)] + pub tools: Option>, + #[serde(default)] + pub tool_choice: Option, + #[serde(default)] + pub stop: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenAiMessage { + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_call_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenAiTool { + #[serde(rename = "type")] + pub tool_type: String, + pub function: OpenAiFunction, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenAiFunction { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenAiToolCall { + pub id: String, + #[serde(rename = "type")] + pub call_type: String, + pub function: OpenAiToolCallFunction, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenAiToolCallFunction { + pub name: String, + pub arguments: String, +} + +// --------------------------------------------------------------------------- +// OpenAI response types (non-streaming) +// --------------------------------------------------------------------------- + +#[derive(Debug, Serialize)] +pub struct OpenAiChatResponse { + pub id: String, + pub object: &'static str, + pub created: u64, + pub model: String, + pub choices: Vec, + pub usage: OpenAiUsage, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiChoice { + pub index: u32, + pub message: OpenAiMessage, + pub finish_reason: String, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiUsage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, +} + +// --------------------------------------------------------------------------- +// OpenAI response types (streaming) +// --------------------------------------------------------------------------- + +#[derive(Debug, Serialize)] +pub struct OpenAiChatChunk { + pub id: String, + pub object: &'static str, + pub created: u64, + pub model: String, + pub choices: Vec, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiChunkChoice { + pub index: u32, + pub delta: OpenAiDelta, + #[serde(skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiToolCallDelta { + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub call_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiToolCallFunctionDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +// --------------------------------------------------------------------------- +// Error response +// --------------------------------------------------------------------------- + +#[derive(Debug, Serialize)] +pub struct OpenAiErrorResponse { + pub error: OpenAiErrorDetail, +} + +#[derive(Debug, Serialize)] +pub struct OpenAiErrorDetail { + pub message: String, + #[serde(rename = "type")] + pub error_type: String, + pub param: Option, + pub code: Option, +} + +// --------------------------------------------------------------------------- +// Conversion functions +// --------------------------------------------------------------------------- + +fn parse_role(s: &str) -> Result { + match s { + "system" => Ok(Role::System), + "user" => Ok(Role::User), + "assistant" => Ok(Role::Assistant), + "tool" => Ok(Role::Tool), + _ => Err(format!("Unknown role: '{}'", s)), + } +} + +pub fn convert_messages(messages: &[OpenAiMessage]) -> Result, String> { + messages + .iter() + .enumerate() + .map(|(i, m)| { + let role = parse_role(&m.role).map_err(|e| format!("messages[{}]: {}", i, e))?; + match role { + Role::Tool => { + let tool_call_id = m.tool_call_id.as_deref().ok_or_else(|| { + format!("messages[{}]: tool message requires 'tool_call_id'", i) + })?; + let name = m + .name + .as_deref() + .ok_or_else(|| format!("messages[{}]: tool message requires 'name'", i))?; + Ok(ChatMessage::tool_result( + tool_call_id, + name, + m.content.as_deref().unwrap_or(""), + )) + } + Role::Assistant => { + if let Some(ref tcs) = m.tool_calls { + let calls: Vec = tcs + .iter() + .map(|tc| ToolCall { + id: tc.id.clone(), + name: tc.function.name.clone(), + arguments: serde_json::from_str(&tc.function.arguments) + .unwrap_or(serde_json::Value::Object(Default::default())), + }) + .collect(); + Ok(ChatMessage::assistant_with_tool_calls( + m.content.clone(), + calls, + )) + } else { + Ok(ChatMessage::assistant(m.content.as_deref().unwrap_or(""))) + } + } + _ => Ok(ChatMessage { + role, + content: m.content.as_deref().unwrap_or("").to_string(), + tool_call_id: None, + name: m.name.clone(), + tool_calls: None, + }), + } + }) + .collect() +} + +pub fn convert_tools(tools: &[OpenAiTool]) -> Vec { + tools + .iter() + .filter(|t| t.tool_type == "function") + .map(|t| ToolDefinition { + name: t.function.name.clone(), + description: t.function.description.clone().unwrap_or_default(), + parameters: t + .function + .parameters + .clone() + .unwrap_or(serde_json::json!({"type": "object", "properties": {}})), + }) + .collect() +} + +fn convert_tool_calls_to_openai(calls: &[ToolCall]) -> Vec { + calls + .iter() + .map(|tc| OpenAiToolCall { + id: tc.id.clone(), + call_type: "function".to_string(), + function: OpenAiToolCallFunction { + name: tc.name.clone(), + arguments: serde_json::to_string(&tc.arguments).unwrap_or_default(), + }, + }) + .collect() +} + +pub fn finish_reason_str(reason: FinishReason) -> String { + match reason { + FinishReason::Stop => "stop".to_string(), + FinishReason::Length => "length".to_string(), + FinishReason::ToolUse => "tool_calls".to_string(), + FinishReason::ContentFilter => "content_filter".to_string(), + FinishReason::Unknown => "stop".to_string(), + } +} + +fn normalize_tool_choice(val: &serde_json::Value) -> Option { + match val { + serde_json::Value::String(s) => Some(s.clone()), + serde_json::Value::Object(obj) => { + // { "type": "function", "function": { "name": "foo" } } → "required" + if obj.contains_key("function") { + Some("required".to_string()) + } else { + obj.get("type") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + } + } + _ => None, + } +} + +fn map_llm_error(err: crate::error::LlmError) -> (StatusCode, Json) { + let (status, error_type, code) = match &err { + crate::error::LlmError::AuthFailed { .. } + | crate::error::LlmError::SessionExpired { .. } => ( + StatusCode::UNAUTHORIZED, + "authentication_error", + "auth_error", + ), + crate::error::LlmError::RateLimited { .. } => ( + StatusCode::TOO_MANY_REQUESTS, + "rate_limit_error", + "rate_limit", + ), + crate::error::LlmError::ContextLengthExceeded { .. } => ( + StatusCode::BAD_REQUEST, + "invalid_request_error", + "context_length_exceeded", + ), + crate::error::LlmError::ModelNotAvailable { .. } => ( + StatusCode::NOT_FOUND, + "invalid_request_error", + "model_not_found", + ), + _ => ( + StatusCode::INTERNAL_SERVER_ERROR, + "server_error", + "internal_error", + ), + }; + + ( + status, + Json(OpenAiErrorResponse { + error: OpenAiErrorDetail { + message: err.to_string(), + error_type: error_type.to_string(), + param: None, + code: Some(code.to_string()), + }, + }), + ) +} + +fn openai_error( + status: StatusCode, + message: impl Into, + error_type: &str, +) -> (StatusCode, Json) { + ( + status, + Json(OpenAiErrorResponse { + error: OpenAiErrorDetail { + message: message.into(), + error_type: error_type.to_string(), + param: None, + code: None, + }, + }), + ) +} + +fn chat_completion_id() -> String { + format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()) +} + +fn unix_timestamp() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Extract stop sequences from the flexible `stop` field. +fn parse_stop(val: &serde_json::Value) -> Option> { + match val { + serde_json::Value::String(s) => Some(vec![s.clone()]), + serde_json::Value::Array(arr) => { + let strs: Vec = arr + .iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(); + if strs.is_empty() { None } else { Some(strs) } + } + _ => None, + } +} + +// --------------------------------------------------------------------------- +// Handlers +// --------------------------------------------------------------------------- + +pub async fn chat_completions_handler( + State(state): State>, + Json(req): Json, +) -> Result)> { + if !state.chat_rate_limiter.check() { + return Err(openai_error( + StatusCode::TOO_MANY_REQUESTS, + "Rate limit exceeded. Please try again later.", + "rate_limit_error", + )); + } + + let llm = state.llm_provider.as_ref().ok_or_else(|| { + openai_error( + StatusCode::SERVICE_UNAVAILABLE, + "LLM provider not configured", + "server_error", + ) + })?; + + if req.messages.is_empty() { + return Err(openai_error( + StatusCode::BAD_REQUEST, + "messages must not be empty", + "invalid_request_error", + )); + } + + // Validate the requested model matches the active model. + // Per-request model switching is not yet supported (see GH issue). + let active_model = llm.active_model_name(); + if req.model != active_model { + return Err(( + StatusCode::NOT_FOUND, + Json(OpenAiErrorResponse { + error: OpenAiErrorDetail { + message: format!( + "Model '{}' not found. The active model is '{}'.", + req.model, active_model + ), + error_type: "invalid_request_error".to_string(), + param: Some("model".to_string()), + code: Some("model_not_found".to_string()), + }, + }), + )); + } + + let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty()); + let stream = req.stream.unwrap_or(false); + + if stream { + return handle_streaming(llm.clone(), req, has_tools) + .await + .map(IntoResponse::into_response); + } + + // --- Non-streaming path --- + + let messages = convert_messages(&req.messages) + .map_err(|e| openai_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; + let model_name = llm.active_model_name(); + let id = chat_completion_id(); + let created = unix_timestamp(); + + if has_tools { + let tools = convert_tools(req.tools.as_deref().unwrap_or(&[])); + let mut tool_req = ToolCompletionRequest::new(messages, tools); + if let Some(t) = req.temperature { + tool_req = tool_req.with_temperature(t); + } + if let Some(mt) = req.max_tokens { + tool_req = tool_req.with_max_tokens(mt); + } + if let Some(ref tc) = req.tool_choice + && let Some(choice) = normalize_tool_choice(tc) + { + tool_req = tool_req.with_tool_choice(choice); + } + + let resp = llm + .complete_with_tools(tool_req) + .await + .map_err(map_llm_error)?; + + let tool_calls_openai = if resp.tool_calls.is_empty() { + None + } else { + Some(convert_tool_calls_to_openai(&resp.tool_calls)) + }; + + let response = OpenAiChatResponse { + id, + object: "chat.completion", + created, + model: model_name, + choices: vec![OpenAiChoice { + index: 0, + message: OpenAiMessage { + role: "assistant".to_string(), + content: resp.content.clone(), + name: None, + tool_call_id: None, + tool_calls: tool_calls_openai, + }, + finish_reason: finish_reason_str(resp.finish_reason), + }], + usage: OpenAiUsage { + prompt_tokens: resp.input_tokens, + completion_tokens: resp.output_tokens, + total_tokens: resp.input_tokens + resp.output_tokens, + }, + }; + + Ok(Json(response).into_response()) + } else { + let mut comp_req = CompletionRequest::new(messages); + if let Some(t) = req.temperature { + comp_req = comp_req.with_temperature(t); + } + if let Some(mt) = req.max_tokens { + comp_req = comp_req.with_max_tokens(mt); + } + if let Some(ref stop_val) = req.stop { + comp_req.stop_sequences = parse_stop(stop_val); + } + + let resp = llm.complete(comp_req).await.map_err(map_llm_error)?; + + let response = OpenAiChatResponse { + id, + object: "chat.completion", + created, + model: model_name, + choices: vec![OpenAiChoice { + index: 0, + message: OpenAiMessage { + role: "assistant".to_string(), + content: Some(resp.content), + name: None, + tool_call_id: None, + tool_calls: None, + }, + finish_reason: finish_reason_str(resp.finish_reason), + }], + usage: OpenAiUsage { + prompt_tokens: resp.input_tokens, + completion_tokens: resp.output_tokens, + total_tokens: resp.input_tokens + resp.output_tokens, + }, + }; + + Ok(Json(response).into_response()) + } +} + +/// Handle streaming responses. +/// +/// The current `LlmProvider` returns complete responses (no streaming method). +/// We execute the LLM call first, then simulate chunked delivery by splitting +/// the response into word-boundary chunks. This ensures LLM failures return +/// proper HTTP errors instead of SSE error events. True token streaming can be +/// added later by extending `LlmProvider` with a `complete_stream()` method. +async fn handle_streaming( + llm: Arc, + req: OpenAiChatRequest, + has_tools: bool, +) -> Result)> { + let messages = convert_messages(&req.messages) + .map_err(|e| openai_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; + + let model_name = llm.active_model_name(); + let id = chat_completion_id(); + let created = unix_timestamp(); + + // Execute the LLM call before starting the SSE stream. + // Since streaming is simulated (LlmProvider returns complete responses), + // this lets us return proper HTTP errors on failure. + enum LlmResult { + Simple(crate::llm::CompletionResponse), + WithTools(crate::llm::ToolCompletionResponse), + } + + let llm_result = if has_tools { + let tools = convert_tools(req.tools.as_deref().unwrap_or(&[])); + let mut tool_req = ToolCompletionRequest::new(messages, tools); + if let Some(t) = req.temperature { + tool_req = tool_req.with_temperature(t); + } + if let Some(mt) = req.max_tokens { + tool_req = tool_req.with_max_tokens(mt); + } + if let Some(ref tc) = req.tool_choice + && let Some(choice) = normalize_tool_choice(tc) + { + tool_req = tool_req.with_tool_choice(choice); + } + LlmResult::WithTools( + llm.complete_with_tools(tool_req) + .await + .map_err(map_llm_error)?, + ) + } else { + let mut comp_req = CompletionRequest::new(messages); + if let Some(t) = req.temperature { + comp_req = comp_req.with_temperature(t); + } + if let Some(mt) = req.max_tokens { + comp_req = comp_req.with_max_tokens(mt); + } + if let Some(ref stop_val) = req.stop { + comp_req.stop_sequences = parse_stop(stop_val); + } + LlmResult::Simple(llm.complete(comp_req).await.map_err(map_llm_error)?) + }; + + // LLM succeeded — emit the response as SSE chunks + let (tx, rx) = tokio::sync::mpsc::channel::>(64); + + tokio::spawn(async move { + // Send initial chunk with role + let role_chunk = OpenAiChatChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: model_name.clone(), + choices: vec![OpenAiChunkChoice { + index: 0, + delta: OpenAiDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: None, + }, + finish_reason: None, + }], + }; + let data = serde_json::to_string(&role_chunk).unwrap_or_default(); + let _ = tx.send(Ok(Event::default().data(data))).await; + + match llm_result { + LlmResult::WithTools(resp) => { + // Stream content chunks + if let Some(ref content) = resp.content { + stream_content_chunks(&tx, &id, created, &model_name, content).await; + } + + // Stream tool calls + if !resp.tool_calls.is_empty() { + let deltas: Vec = resp + .tool_calls + .iter() + .enumerate() + .map(|(i, tc)| OpenAiToolCallDelta { + index: i as u32, + id: Some(tc.id.clone()), + call_type: Some("function".to_string()), + function: Some(OpenAiToolCallFunctionDelta { + name: Some(tc.name.clone()), + arguments: Some( + serde_json::to_string(&tc.arguments).unwrap_or_default(), + ), + }), + }) + .collect(); + + let chunk = OpenAiChatChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: model_name.clone(), + choices: vec![OpenAiChunkChoice { + index: 0, + delta: OpenAiDelta { + role: None, + content: None, + tool_calls: Some(deltas), + }, + finish_reason: None, + }], + }; + let data = serde_json::to_string(&chunk).unwrap_or_default(); + let _ = tx.send(Ok(Event::default().data(data))).await; + } + + // Final chunk with finish_reason + send_finish_chunk(&tx, &id, created, &model_name, resp.finish_reason).await; + } + LlmResult::Simple(resp) => { + stream_content_chunks(&tx, &id, created, &model_name, &resp.content).await; + send_finish_chunk(&tx, &id, created, &model_name, resp.finish_reason).await; + } + } + + // Send [DONE] sentinel + let _ = tx.send(Ok(Event::default().data("[DONE]"))).await; + }); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx); + let sse = Sse::new(stream).keep_alive(KeepAlive::new().text("")); + let mut response = sse.into_response(); + response.headers_mut().insert( + "x-ironclaw-streaming", + HeaderValue::from_static("simulated"), + ); + Ok(response) +} + +/// Split content into word-boundary chunks and send as SSE events. +async fn stream_content_chunks( + tx: &tokio::sync::mpsc::Sender>, + id: &str, + created: u64, + model: &str, + content: &str, +) { + // Split on word boundaries, grouping ~20 chars per chunk + let mut buf = String::new(); + for word in content.split_inclusive(char::is_whitespace) { + buf.push_str(word); + if buf.len() >= 20 { + let chunk = OpenAiChatChunk { + id: id.to_string(), + object: "chat.completion.chunk", + created, + model: model.to_string(), + choices: vec![OpenAiChunkChoice { + index: 0, + delta: OpenAiDelta { + role: None, + content: Some(buf.clone()), + tool_calls: None, + }, + finish_reason: None, + }], + }; + let data = serde_json::to_string(&chunk).unwrap_or_default(); + if tx.send(Ok(Event::default().data(data))).await.is_err() { + return; + } + buf.clear(); + } + } + // Flush remaining + if !buf.is_empty() { + let chunk = OpenAiChatChunk { + id: id.to_string(), + object: "chat.completion.chunk", + created, + model: model.to_string(), + choices: vec![OpenAiChunkChoice { + index: 0, + delta: OpenAiDelta { + role: None, + content: Some(buf), + tool_calls: None, + }, + finish_reason: None, + }], + }; + let data = serde_json::to_string(&chunk).unwrap_or_default(); + let _ = tx.send(Ok(Event::default().data(data))).await; + } +} + +async fn send_finish_chunk( + tx: &tokio::sync::mpsc::Sender>, + id: &str, + created: u64, + model: &str, + reason: FinishReason, +) { + let chunk = OpenAiChatChunk { + id: id.to_string(), + object: "chat.completion.chunk", + created, + model: model.to_string(), + choices: vec![OpenAiChunkChoice { + index: 0, + delta: OpenAiDelta { + role: None, + content: None, + tool_calls: None, + }, + finish_reason: Some(finish_reason_str(reason)), + }], + }; + let data = serde_json::to_string(&chunk).unwrap_or_default(); + let _ = tx.send(Ok(Event::default().data(data))).await; +} + +pub async fn models_handler( + State(state): State>, +) -> Result, (StatusCode, Json)> { + let llm = state.llm_provider.as_ref().ok_or_else(|| { + openai_error( + StatusCode::SERVICE_UNAVAILABLE, + "LLM provider not configured", + "server_error", + ) + })?; + + let model_name = llm.active_model_name(); + let created = unix_timestamp(); + + // Try to fetch available models from the provider + let models = match llm.list_models().await { + Ok(names) if !names.is_empty() => names + .into_iter() + .map(|name| { + serde_json::json!({ + "id": name, + "object": "model", + "created": created, + "owned_by": "ironclaw" + }) + }) + .collect(), + Ok(_) => { + // Empty list: fall back to active model + vec![serde_json::json!({ + "id": model_name, + "object": "model", + "created": created, + "owned_by": "ironclaw" + })] + } + Err(e) => return Err(map_llm_error(e)), + }; + + Ok(Json(serde_json::json!({ + "object": "list", + "data": models + }))) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_role() { + assert_eq!(parse_role("system").unwrap(), Role::System); + assert_eq!(parse_role("user").unwrap(), Role::User); + assert_eq!(parse_role("assistant").unwrap(), Role::Assistant); + assert_eq!(parse_role("tool").unwrap(), Role::Tool); + } + + #[test] + fn test_parse_role_unknown_rejected() { + let err = parse_role("unknown").unwrap_err(); + assert!(err.contains("Unknown role")); + assert!(err.contains("unknown")); + } + + #[test] + fn test_finish_reason_str() { + assert_eq!(finish_reason_str(FinishReason::Stop), "stop"); + assert_eq!(finish_reason_str(FinishReason::Length), "length"); + assert_eq!(finish_reason_str(FinishReason::ToolUse), "tool_calls"); + assert_eq!( + finish_reason_str(FinishReason::ContentFilter), + "content_filter" + ); + assert_eq!(finish_reason_str(FinishReason::Unknown), "stop"); + } + + #[test] + fn test_convert_messages_basic() { + let msgs = vec![ + OpenAiMessage { + role: "system".to_string(), + content: Some("You are helpful.".to_string()), + name: None, + tool_call_id: None, + tool_calls: None, + }, + OpenAiMessage { + role: "user".to_string(), + content: Some("Hello".to_string()), + name: None, + tool_call_id: None, + tool_calls: None, + }, + ]; + + let converted = convert_messages(&msgs).unwrap(); + assert_eq!(converted.len(), 2); + assert_eq!(converted[0].role, Role::System); + assert_eq!(converted[0].content, "You are helpful."); + assert_eq!(converted[1].role, Role::User); + assert_eq!(converted[1].content, "Hello"); + } + + #[test] + fn test_convert_messages_with_tool_results() { + let msgs = vec![OpenAiMessage { + role: "tool".to_string(), + content: Some("42".to_string()), + name: Some("calculator".to_string()), + tool_call_id: Some("call_123".to_string()), + tool_calls: None, + }]; + + let converted = convert_messages(&msgs).unwrap(); + assert_eq!(converted.len(), 1); + assert_eq!(converted[0].role, Role::Tool); + assert_eq!(converted[0].content, "42"); + assert_eq!(converted[0].tool_call_id.as_deref(), Some("call_123")); + assert_eq!(converted[0].name.as_deref(), Some("calculator")); + } + + #[test] + fn test_convert_tools() { + let tools = vec![OpenAiTool { + tool_type: "function".to_string(), + function: OpenAiFunction { + name: "get_weather".to_string(), + description: Some("Get weather for a location".to_string()), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": { + "location": { "type": "string" } + }, + "required": ["location"] + })), + }, + }]; + + let converted = convert_tools(&tools); + assert_eq!(converted.len(), 1); + assert_eq!(converted[0].name, "get_weather"); + assert_eq!(converted[0].description, "Get weather for a location"); + } + + #[test] + fn test_convert_tool_calls_to_openai() { + let calls = vec![ToolCall { + id: "call_abc".to_string(), + name: "search".to_string(), + arguments: serde_json::json!({"query": "rust"}), + }]; + + let converted = convert_tool_calls_to_openai(&calls); + assert_eq!(converted.len(), 1); + assert_eq!(converted[0].id, "call_abc"); + assert_eq!(converted[0].call_type, "function"); + assert_eq!(converted[0].function.name, "search"); + assert!(converted[0].function.arguments.contains("rust")); + } + + #[test] + fn test_normalize_tool_choice() { + // String variant + let v = serde_json::json!("auto"); + assert_eq!(normalize_tool_choice(&v), Some("auto".to_string())); + + // Object with function + let v = serde_json::json!({"type": "function", "function": {"name": "foo"}}); + assert_eq!(normalize_tool_choice(&v), Some("required".to_string())); + + // Object with type only + let v = serde_json::json!({"type": "none"}); + assert_eq!(normalize_tool_choice(&v), Some("none".to_string())); + + // Null + let v = serde_json::Value::Null; + assert_eq!(normalize_tool_choice(&v), None); + } + + #[test] + fn test_openai_request_deserialize_minimal() { + let json = r#"{"model":"gpt-4","messages":[{"role":"user","content":"Hi"}]}"#; + let req: OpenAiChatRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.model, "gpt-4"); + assert_eq!(req.messages.len(), 1); + assert_eq!(req.stream, None); + assert_eq!(req.temperature, None); + } + + #[test] + fn test_openai_request_deserialize_streaming() { + let json = r#"{"model":"gpt-4","messages":[{"role":"user","content":"Hi"}],"stream":true,"temperature":0.7}"#; + let req: OpenAiChatRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.stream, Some(true)); + assert_eq!(req.temperature, Some(0.7)); + } + + #[test] + fn test_openai_response_serialize() { + let resp = OpenAiChatResponse { + id: "chatcmpl-test".to_string(), + object: "chat.completion", + created: 1234567890, + model: "test-model".to_string(), + choices: vec![OpenAiChoice { + index: 0, + message: OpenAiMessage { + role: "assistant".to_string(), + content: Some("Hello!".to_string()), + name: None, + tool_call_id: None, + tool_calls: None, + }, + finish_reason: "stop".to_string(), + }], + usage: OpenAiUsage { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, + }; + + let json = serde_json::to_value(&resp).unwrap(); + assert_eq!(json["object"], "chat.completion"); + assert_eq!(json["choices"][0]["finish_reason"], "stop"); + assert_eq!(json["choices"][0]["message"]["content"], "Hello!"); + assert_eq!(json["usage"]["total_tokens"], 15); + } + + #[test] + fn test_openai_message_with_null_content() { + let json = r#"{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"search","arguments":"{\"q\":\"test\"}"}}]}"#; + let msg: OpenAiMessage = serde_json::from_str(json).unwrap(); + assert_eq!(msg.role, "assistant"); + assert!(msg.content.is_none()); + assert!(msg.tool_calls.is_some()); + assert_eq!(msg.tool_calls.as_ref().unwrap().len(), 1); + } + + #[test] + fn test_convert_messages_unknown_role_rejected() { + let msgs = vec![OpenAiMessage { + role: "moderator".to_string(), + content: Some("Hi".to_string()), + name: None, + tool_call_id: None, + tool_calls: None, + }]; + let err = convert_messages(&msgs).unwrap_err(); + assert!(err.contains("messages[0]")); + assert!(err.contains("Unknown role")); + } + + #[test] + fn test_convert_messages_tool_missing_fields() { + // Missing tool_call_id + let msgs = vec![OpenAiMessage { + role: "tool".to_string(), + content: Some("result".to_string()), + name: Some("calc".to_string()), + tool_call_id: None, + tool_calls: None, + }]; + let err = convert_messages(&msgs).unwrap_err(); + assert!(err.contains("tool_call_id")); + + // Missing name + let msgs = vec![OpenAiMessage { + role: "tool".to_string(), + content: Some("result".to_string()), + name: None, + tool_call_id: Some("call_1".to_string()), + tool_calls: None, + }]; + let err = convert_messages(&msgs).unwrap_err(); + assert!(err.contains("'name'")); + } + + #[test] + fn test_parse_stop_string() { + let v = serde_json::json!("STOP"); + assert_eq!(parse_stop(&v), Some(vec!["STOP".to_string()])); + } + + #[test] + fn test_parse_stop_array() { + let v = serde_json::json!(["STOP", "END"]); + assert_eq!( + parse_stop(&v), + Some(vec!["STOP".to_string(), "END".to_string()]) + ); + } + + #[test] + fn test_parse_stop_null() { + let v = serde_json::Value::Null; + assert_eq!(parse_stop(&v), None); + } +} diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index 11363dfa..744c369e 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -5,10 +5,11 @@ use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use axum::{ Json, Router, - extract::{Path, Query, State, WebSocketUpgrade}, + extract::{DefaultBodyLimit, Path, Query, State, WebSocketUpgrade}, http::{StatusCode, header}, middleware, response::{ @@ -20,6 +21,7 @@ use axum::{ use serde::Deserialize; use tokio::sync::{mpsc, oneshot}; use tokio_stream::StreamExt; +use tower_http::cors::{AllowHeaders, CorsLayer}; use uuid::Uuid; use crate::agent::SessionManager; @@ -28,11 +30,85 @@ use crate::channels::web::auth::{AuthState, auth_middleware}; use crate::channels::web::log_layer::LogBroadcaster; use crate::channels::web::sse::SseManager; use crate::channels::web::types::*; -use crate::context::ContextManager; +use crate::db::Database; use crate::extensions::ExtensionManager; +use crate::orchestrator::job_manager::ContainerJobManager; use crate::tools::ToolRegistry; use crate::workspace::Workspace; +/// Shared prompt queue: maps job IDs to pending follow-up prompts for Claude Code bridges. +pub type PromptQueue = Arc< + tokio::sync::Mutex< + std::collections::HashMap< + uuid::Uuid, + std::collections::VecDeque, + >, + >, +>; + +/// Simple sliding-window rate limiter. +/// +/// Tracks the number of requests in the current window. Resets when the window expires. +/// Not per-IP (since this is a single-user gateway with auth), but prevents flooding. +pub struct RateLimiter { + /// Requests remaining in the current window. + remaining: AtomicU64, + /// Epoch second when the current window started. + window_start: AtomicU64, + /// Maximum requests per window. + max_requests: u64, + /// Window duration in seconds. + window_secs: u64, +} + +impl RateLimiter { + pub fn new(max_requests: u64, window_secs: u64) -> Self { + Self { + remaining: AtomicU64::new(max_requests), + window_start: AtomicU64::new( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + ), + max_requests, + window_secs, + } + } + + /// Try to consume one request. Returns `true` if allowed, `false` if rate limited. + pub fn check(&self) -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + let window = self.window_start.load(Ordering::Relaxed); + if now.saturating_sub(window) >= self.window_secs { + // Window expired, reset + self.window_start.store(now, Ordering::Relaxed); + self.remaining + .store(self.max_requests - 1, Ordering::Relaxed); + return true; + } + + // Try to decrement remaining + loop { + let current = self.remaining.load(Ordering::Relaxed); + if current == 0 { + return false; + } + if self + .remaining + .compare_exchange_weak(current, current - 1, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + return true; + } + } + } +} + /// Shared state for all gateway handlers. pub struct GatewayState { /// Channel to send messages to the agent loop. @@ -41,8 +117,6 @@ pub struct GatewayState { pub sse: SseManager, /// Workspace for memory API. pub workspace: Option>, - /// Context manager for jobs API. - pub context_manager: Option>, /// Session manager for thread info. pub session_manager: Option>, /// Log broadcaster for the logs SSE endpoint. @@ -51,12 +125,22 @@ pub struct GatewayState { pub extension_manager: Option>, /// Tool registry for listing registered tools. pub tool_registry: Option>, + /// Database store for sandbox job persistence. + pub store: Option>, + /// Container job manager for sandbox operations. + pub job_manager: Option>, + /// Prompt queue for Claude Code follow-up prompts. + pub prompt_queue: Option, /// User ID for this gateway. pub user_id: String, /// Shutdown signal sender. pub shutdown_tx: tokio::sync::RwLock>>, /// WebSocket connection tracker. pub ws_tracker: Option>, + /// LLM provider for OpenAI-compatible API proxy. + pub llm_provider: Option>, + /// Rate limiter for chat endpoints (30 messages per 60 seconds). + pub chat_rate_limiter: RateLimiter, } /// Start the gateway HTTP server. @@ -90,6 +174,8 @@ pub async fn start_server( // Chat .route("/api/chat/send", post(chat_send_handler)) .route("/api/chat/approval", post(chat_approval_handler)) + .route("/api/chat/auth-token", post(chat_auth_token_handler)) + .route("/api/chat/auth-cancel", post(chat_auth_cancel_handler)) .route("/api/chat/events", get(chat_events_handler)) .route("/api/chat/ws", get(chat_ws_handler)) .route("/api/chat/history", get(chat_history_handler)) @@ -106,6 +192,11 @@ pub async fn start_server( .route("/api/jobs/summary", get(jobs_summary_handler)) .route("/api/jobs/{id}", get(jobs_detail_handler)) .route("/api/jobs/{id}/cancel", post(jobs_cancel_handler)) + .route("/api/jobs/{id}/restart", post(jobs_restart_handler)) + .route("/api/jobs/{id}/prompt", post(jobs_prompt_handler)) + .route("/api/jobs/{id}/events", get(jobs_events_handler)) + .route("/api/jobs/{id}/files/list", get(job_files_list_handler)) + .route("/api/jobs/{id}/files/read", get(job_files_read_handler)) // Logs .route("/api/logs/events", get(logs_events_handler)) // Extensions @@ -120,9 +211,42 @@ pub async fn start_server( "/api/extensions/{name}/remove", post(extensions_remove_handler), ) + // Routines + .route("/api/routines", get(routines_list_handler)) + .route("/api/routines/summary", get(routines_summary_handler)) + .route("/api/routines/{id}", get(routines_detail_handler)) + .route("/api/routines/{id}/trigger", post(routines_trigger_handler)) + .route("/api/routines/{id}/toggle", post(routines_toggle_handler)) + .route( + "/api/routines/{id}", + axum::routing::delete(routines_delete_handler), + ) + .route("/api/routines/{id}/runs", get(routines_runs_handler)) + // Settings + .route("/api/settings", get(settings_list_handler)) + .route("/api/settings/export", get(settings_export_handler)) + .route("/api/settings/import", post(settings_import_handler)) + .route("/api/settings/{key}", get(settings_get_handler)) + .route( + "/api/settings/{key}", + axum::routing::put(settings_set_handler), + ) + .route( + "/api/settings/{key}", + axum::routing::delete(settings_delete_handler), + ) // Gateway control plane .route("/api/gateway/status", get(gateway_status_handler)) - .route_layer(middleware::from_fn_with_state(auth_state, auth_middleware)); + // OpenAI-compatible API + .route( + "/v1/chat/completions", + post(super::openai_compat::chat_completions_handler), + ) + .route("/v1/models", get(super::openai_compat::models_handler)) + .route_layer(middleware::from_fn_with_state( + auth_state.clone(), + auth_middleware, + )); // Static file routes (no auth, served from embedded strings) let statics = Router::new() @@ -130,10 +254,46 @@ pub async fn start_server( .route("/style.css", get(css_handler)) .route("/app.js", get(js_handler)); + // Project file serving (behind auth to prevent unauthorized file access). + let projects = Router::new() + .route("/projects/{project_id}", get(project_redirect_handler)) + .route("/projects/{project_id}/", get(project_index_handler)) + .route("/projects/{project_id}/{*path}", get(project_file_handler)) + .route_layer(middleware::from_fn_with_state( + auth_state.clone(), + auth_middleware, + )); + + // CORS: restrict to same-origin by default. Only localhost/127.0.0.1 + // origins are allowed, since the gateway is a local-first service. + let cors = CorsLayer::new() + .allow_origin([ + format!("http://{}:{}", addr.ip(), addr.port()) + .parse() + .expect("valid origin"), + format!("http://localhost:{}", addr.port()) + .parse() + .expect("valid origin"), + ]) + .allow_methods([ + axum::http::Method::GET, + axum::http::Method::POST, + axum::http::Method::PUT, + axum::http::Method::DELETE, + ]) + .allow_headers(AllowHeaders::list([ + header::CONTENT_TYPE, + header::AUTHORIZATION, + ])) + .allow_credentials(true); + let app = Router::new() .merge(public) .merge(statics) + .merge(projects) .merge(protected) + .layer(cors) + .layer(DefaultBodyLimit::max(1024 * 1024)) // 1 MB max request body .with_state(state.clone()); let (shutdown_tx, shutdown_rx) = oneshot::channel(); @@ -189,10 +349,18 @@ async fn chat_send_handler( State(state): State>, Json(req): Json, ) -> Result<(StatusCode, Json), (StatusCode, String)> { + if !state.chat_rate_limiter.check() { + return Err(( + StatusCode::TOO_MANY_REQUESTS, + "Rate limit exceeded. Try again shortly.".to_string(), + )); + } + let mut msg = IncomingMessage::new("gateway", &state.user_id, &req.content); if let Some(ref thread_id) = req.thread_id { msg = msg.with_thread(thread_id); + msg = msg.with_metadata(serde_json::json!({"thread_id": thread_id})); } let msg_id = msg.id; @@ -256,7 +424,12 @@ async fn chat_approval_handler( ) })?; - let msg = IncomingMessage::new("gateway", &state.user_id, content); + let mut msg = IncomingMessage::new("gateway", &state.user_id, content); + + if let Some(ref thread_id) = req.thread_id { + msg = msg.with_thread(thread_id); + } + let msg_id = msg.id; let tx_guard = state.msg_tx.read().await; @@ -281,21 +454,136 @@ async fn chat_approval_handler( )) } -async fn chat_events_handler(State(state): State>) -> impl IntoResponse { - // subscribe() returns Sse> so no lifetime issues - state.sse.subscribe() +/// Submit an auth token directly to the extension manager, bypassing the message pipeline. +/// +/// The token never touches the LLM, chat history, or SSE stream. +async fn chat_auth_token_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Extension manager not available".to_string(), + ))?; + + let result = ext_mgr + .auth(&req.extension_name, Some(&req.token)) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if result.status == "authenticated" { + // Auto-activate so tools are available immediately + let msg = match ext_mgr.activate(&req.extension_name).await { + Ok(r) => format!( + "{} authenticated ({} tools loaded)", + req.extension_name, + r.tools_loaded.len() + ), + Err(e) => format!( + "{} authenticated but activation failed: {}", + req.extension_name, e + ), + }; + + // Clear auth mode on the active thread + clear_auth_mode(&state).await; + + state.sse.broadcast(SseEvent::AuthCompleted { + extension_name: req.extension_name, + success: true, + message: msg.clone(), + }); + + Ok(Json(ActionResponse::ok(msg))) + } else { + // Re-emit auth_required for retry + state.sse.broadcast(SseEvent::AuthRequired { + extension_name: req.extension_name.clone(), + instructions: result.instructions.clone(), + auth_url: result.auth_url.clone(), + setup_url: result.setup_url.clone(), + }); + Ok(Json(ActionResponse::fail( + result + .instructions + .unwrap_or_else(|| "Invalid token".to_string()), + ))) + } +} + +/// Cancel an in-progress auth flow. +async fn chat_auth_cancel_handler( + State(state): State>, + Json(_req): Json, +) -> Result, (StatusCode, String)> { + clear_auth_mode(&state).await; + Ok(Json(ActionResponse::ok("Auth cancelled"))) +} + +/// Clear pending auth mode on the active thread. +pub async fn clear_auth_mode(state: &GatewayState) { + if let Some(ref sm) = state.session_manager { + let session = sm.get_or_create_session(&state.user_id).await; + let mut sess = session.lock().await; + if let Some(thread_id) = sess.active_thread + && let Some(thread) = sess.threads.get_mut(&thread_id) + { + thread.pending_auth = None; + } + } +} + +async fn chat_events_handler( + State(state): State>, +) -> Result { + state.sse.subscribe().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Too many connections".to_string(), + )) } async fn chat_ws_handler( + headers: axum::http::HeaderMap, ws: WebSocketUpgrade, State(state): State>, -) -> impl IntoResponse { - ws.on_upgrade(move |socket| crate::channels::web::ws::handle_ws_connection(socket, state)) +) -> Result { + // Validate Origin header to prevent cross-site WebSocket hijacking. + // Require the header outright; browsers always send it for WS upgrades, + // so a missing Origin means a non-browser client trying to bypass the check. + let origin = headers + .get("origin") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + ( + StatusCode::FORBIDDEN, + "WebSocket Origin header required".to_string(), + ) + })?; + + // Extract the host from the origin and compare exactly, so that + // crafted origins like "http://localhost.evil.com" are rejected. + // Origin format is "scheme://host[:port]". + let host = origin + .strip_prefix("http://") + .or_else(|| origin.strip_prefix("https://")) + .and_then(|rest| rest.split(':').next()?.split('/').next()) + .unwrap_or(""); + + let is_local = matches!(host, "localhost" | "127.0.0.1" | "[::1]"); + if !is_local { + return Err(( + StatusCode::FORBIDDEN, + "WebSocket origin not allowed".to_string(), + )); + } + Ok(ws.on_upgrade(move |socket| crate::channels::web::ws::handle_ws_connection(socket, state))) } #[derive(Deserialize)] struct HistoryQuery { thread_id: Option, + limit: Option, + before: Option, } async fn chat_history_handler( @@ -310,6 +598,22 @@ async fn chat_history_handler( let session = session_manager.get_or_create_session(&state.user_id).await; let sess = session.lock().await; + let limit = query.limit.unwrap_or(50); + let before_cursor = query + .before + .as_deref() + .map(|s| { + chrono::DateTime::parse_from_rfc3339(s) + .map(|dt| dt.with_timezone(&chrono::Utc)) + .map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid 'before' timestamp".to_string(), + ) + }) + }) + .transpose()?; + // Find the thread let thread_id = if let Some(ref tid) = query.thread_id { Uuid::parse_str(tid) @@ -319,34 +623,140 @@ async fn chat_history_handler( .ok_or((StatusCode::NOT_FOUND, "No active thread".to_string()))? }; - let thread = sess - .threads - .get(&thread_id) - .ok_or((StatusCode::NOT_FOUND, "Thread not found".to_string()))?; + // Verify the thread belongs to the authenticated user before returning any data. + // In-memory threads are already scoped by user via session_manager, but DB + // lookups could expose another user's conversation if the UUID is guessed. + if query.thread_id.is_some() + && let Some(ref store) = state.store + { + let owned = store + .conversation_belongs_to_user(thread_id, &state.user_id) + .await + .unwrap_or(false); + if !owned && !sess.threads.contains_key(&thread_id) { + return Err((StatusCode::NOT_FOUND, "Thread not found".to_string())); + } + } - let turns: Vec = thread - .turns - .iter() - .map(|t| TurnInfo { - turn_number: t.turn_number, - user_input: t.user_input.clone(), - response: t.response.clone(), - state: format!("{:?}", t.state), - started_at: t.started_at.to_rfc3339(), - completed_at: t.completed_at.map(|dt| dt.to_rfc3339()), - tool_calls: t - .tool_calls - .iter() - .map(|tc| ToolCallInfo { - name: tc.name.clone(), - has_result: tc.result.is_some(), - has_error: tc.error.is_some(), - }) - .collect(), - }) - .collect(); + // For paginated requests (before cursor set), always go to DB + if before_cursor.is_some() + && let Some(ref store) = state.store + { + let (messages, has_more) = store + .list_conversation_messages_paginated(thread_id, before_cursor, limit as i64) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - Ok(Json(HistoryResponse { thread_id, turns })) + let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339()); + let turns = build_turns_from_db_messages(&messages); + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more, + oldest_timestamp, + })); + } + + // Try in-memory first (freshest data for active threads) + if let Some(thread) = sess.threads.get(&thread_id) + && !thread.turns.is_empty() + { + let turns: Vec = thread + .turns + .iter() + .map(|t| TurnInfo { + turn_number: t.turn_number, + user_input: t.user_input.clone(), + response: t.response.clone(), + state: format!("{:?}", t.state), + started_at: t.started_at.to_rfc3339(), + completed_at: t.completed_at.map(|dt| dt.to_rfc3339()), + tool_calls: t + .tool_calls + .iter() + .map(|tc| ToolCallInfo { + name: tc.name.clone(), + has_result: tc.result.is_some(), + has_error: tc.error.is_some(), + }) + .collect(), + }) + .collect(); + + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more: false, + oldest_timestamp: None, + })); + } + + // Fall back to DB for historical threads not in memory (paginated) + if let Some(ref store) = state.store { + let (messages, has_more) = store + .list_conversation_messages_paginated(thread_id, None, limit as i64) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if !messages.is_empty() { + let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339()); + let turns = build_turns_from_db_messages(&messages); + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more, + oldest_timestamp, + })); + } + } + + // Empty thread (just created, no messages yet) + Ok(Json(HistoryResponse { + thread_id, + turns: Vec::new(), + has_more: false, + oldest_timestamp: None, + })) +} + +/// Build TurnInfo pairs from flat DB messages (alternating user/assistant). +fn build_turns_from_db_messages(messages: &[crate::history::ConversationMessage]) -> Vec { + let mut turns = Vec::new(); + let mut turn_number = 0; + let mut iter = messages.iter().peekable(); + + while let Some(msg) = iter.next() { + if msg.role == "user" { + let mut turn = TurnInfo { + turn_number, + user_input: msg.content.clone(), + response: None, + state: "Completed".to_string(), + started_at: msg.created_at.to_rfc3339(), + completed_at: None, + tool_calls: Vec::new(), + }; + + // Check if next message is an assistant response + if let Some(next) = iter.peek() + && next.role == "assistant" + { + let assistant_msg = iter.next().expect("peeked"); + turn.response = Some(assistant_msg.content.clone()); + turn.completed_at = Some(assistant_msg.created_at.to_rfc3339()); + } + + // Incomplete turn (user message without response) + if turn.response.is_none() { + turn.state = "Failed".to_string(); + } + + turns.push(turn); + turn_number += 1; + } + } + + turns } async fn chat_threads_handler( @@ -360,6 +770,61 @@ async fn chat_threads_handler( let session = session_manager.get_or_create_session(&state.user_id).await; let sess = session.lock().await; + // Try DB first for persistent thread list + if let Some(ref store) = state.store { + // Auto-create assistant thread if it doesn't exist + let assistant_id = store + .get_or_create_assistant_conversation(&state.user_id, "gateway") + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if let Ok(summaries) = store + .list_conversations_with_preview(&state.user_id, "gateway", 50) + .await + { + let mut assistant_thread = None; + let mut threads = Vec::new(); + + for s in &summaries { + let info = ThreadInfo { + id: s.id, + state: "Idle".to_string(), + turn_count: (s.message_count / 2).max(0) as usize, + created_at: s.started_at.to_rfc3339(), + updated_at: s.last_activity.to_rfc3339(), + title: s.title.clone(), + thread_type: s.thread_type.clone(), + }; + + if s.id == assistant_id { + assistant_thread = Some(info); + } else { + threads.push(info); + } + } + + // If assistant wasn't in the list (0 messages), synthesize it + if assistant_thread.is_none() { + assistant_thread = Some(ThreadInfo { + id: assistant_id, + state: "Idle".to_string(), + turn_count: 0, + created_at: chrono::Utc::now().to_rfc3339(), + updated_at: chrono::Utc::now().to_rfc3339(), + title: None, + thread_type: Some("assistant".to_string()), + }); + } + + return Ok(Json(ThreadListResponse { + assistant_thread, + threads, + active_thread: sess.active_thread, + })); + } + } + + // Fallback: in-memory only (no assistant thread without DB) let threads: Vec = sess .threads .values() @@ -369,10 +834,13 @@ async fn chat_threads_handler( turn_count: t.turns.len(), created_at: t.created_at.to_rfc3339(), updated_at: t.updated_at.to_rfc3339(), + title: None, + thread_type: None, }) .collect(); Ok(Json(ThreadListResponse { + assistant_thread: None, threads, active_thread: sess.active_thread, })) @@ -389,14 +857,39 @@ async fn chat_new_thread_handler( let session = session_manager.get_or_create_session(&state.user_id).await; let mut sess = session.lock().await; let thread = sess.create_thread(); - - Ok(Json(ThreadInfo { + let thread_id = thread.id; + let info = ThreadInfo { id: thread.id, state: format!("{:?}", thread.state), turn_count: thread.turns.len(), created_at: thread.created_at.to_rfc3339(), updated_at: thread.updated_at.to_rfc3339(), - })) + title: None, + thread_type: Some("thread".to_string()), + }; + + // Persist the empty conversation row with thread_type metadata + if let Some(ref store) = state.store { + let store = Arc::clone(store); + let user_id = state.user_id.clone(); + tokio::spawn(async move { + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", &user_id, None) + .await + { + tracing::warn!("Failed to persist new thread: {}", e); + } + let metadata_val = serde_json::json!("thread"); + if let Err(e) = store + .update_conversation_metadata_field(thread_id, "thread_type", &metadata_val) + .await + { + tracing::warn!("Failed to set thread_type metadata: {}", e); + } + }); + } + + Ok(Json(info)) } // --- Memory handlers --- @@ -564,26 +1057,40 @@ async fn memory_search_handler( async fn jobs_list_handler( State(state): State>, ) -> Result, (StatusCode, String)> { - let context_manager = state.context_manager.as_ref().ok_or(( + let store = state.store.as_ref().ok_or(( StatusCode::SERVICE_UNAVAILABLE, - "Context manager not available".to_string(), + "Database not available".to_string(), ))?; - let job_ids = context_manager.all_jobs_for(&state.user_id).await; - let mut jobs = Vec::new(); + // Fetch sandbox jobs scoped to the authenticated user. + let sandbox_jobs = store + .list_sandbox_jobs_for_user(&state.user_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - for job_id in job_ids { - if let Ok(ctx) = context_manager.get_context(job_id).await { - jobs.push(JobInfo { - id: ctx.job_id, - title: ctx.title.clone(), - state: ctx.state.to_string(), - user_id: ctx.user_id.clone(), - created_at: ctx.created_at.to_rfc3339(), - started_at: ctx.started_at.map(|dt| dt.to_rfc3339()), - }); - } - } + // Scope jobs to the authenticated user. + let mut jobs: Vec = sandbox_jobs + .iter() + .filter(|j| j.user_id == state.user_id) + .map(|j| { + let ui_state = match j.status.as_str() { + "creating" => "pending", + "running" => "in_progress", + s => s, + }; + JobInfo { + id: j.id, + title: j.task.clone(), + state: ui_state.to_string(), + user_id: j.user_id.clone(), + created_at: j.created_at.to_rfc3339(), + started_at: j.started_at.map(|dt| dt.to_rfc3339()), + } + }) + .collect(); + + // Most recent first. + jobs.sort_by(|a, b| b.created_at.cmp(&a.created_at)); Ok(Json(JobListResponse { jobs })) } @@ -591,87 +1098,446 @@ async fn jobs_list_handler( async fn jobs_summary_handler( State(state): State>, ) -> Result, (StatusCode, String)> { - let context_manager = state.context_manager.as_ref().ok_or(( + let store = state.store.as_ref().ok_or(( StatusCode::SERVICE_UNAVAILABLE, - "Context manager not available".to_string(), + "Database not available".to_string(), ))?; - let summary = context_manager.summary_for(&state.user_id).await; + let s = store + .sandbox_job_summary_for_user(&state.user_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; Ok(Json(JobSummaryResponse { - total: summary.total, - pending: summary.pending, - in_progress: summary.in_progress, - completed: summary.completed, - failed: summary.failed, - stuck: summary.stuck, + total: s.total, + pending: s.creating, + in_progress: s.running, + completed: s.completed, + failed: s.failed + s.interrupted, + stuck: 0, })) } async fn jobs_detail_handler( State(state): State>, Path(id): Path, -) -> Result, (StatusCode, String)> { - let context_manager = state.context_manager.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Context manager not available".to_string(), - ))?; - +) -> Result, (StatusCode, String)> { let job_id = Uuid::parse_str(&id) .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - let ctx = context_manager - .get_context(job_id) - .await - .map_err(|_| (StatusCode::NOT_FOUND, "Job not found".to_string()))?; + // Try sandbox job from DB first, scoped to the authenticated user. + if let Some(ref store) = state.store + && let Ok(Some(job)) = store.get_sandbox_job(job_id).await + { + if job.user_id != state.user_id { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + let browse_id = std::path::Path::new(&job.project_dir) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| job.id.to_string()); - if ctx.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + let ui_state = match job.status.as_str() { + "creating" => "pending", + "running" => "in_progress", + s => s, + }; + + let elapsed_secs = job.started_at.map(|start| { + let end = job.completed_at.unwrap_or_else(chrono::Utc::now); + (end - start).num_seconds().max(0) as u64 + }); + + // Synthesize transitions from timestamps. + let mut transitions = Vec::new(); + if let Some(started) = job.started_at { + transitions.push(TransitionInfo { + from: "creating".to_string(), + to: "running".to_string(), + timestamp: started.to_rfc3339(), + reason: None, + }); + } + if let Some(completed) = job.completed_at { + transitions.push(TransitionInfo { + from: "running".to_string(), + to: job.status.clone(), + timestamp: completed.to_rfc3339(), + reason: job.failure_reason.clone(), + }); + } + + return Ok(Json(JobDetailResponse { + id: job.id, + title: job.task.clone(), + description: String::new(), + state: ui_state.to_string(), + user_id: job.user_id.clone(), + created_at: job.created_at.to_rfc3339(), + started_at: job.started_at.map(|dt| dt.to_rfc3339()), + completed_at: job.completed_at.map(|dt| dt.to_rfc3339()), + elapsed_secs, + project_dir: Some(job.project_dir.clone()), + browse_url: Some(format!("/projects/{}/", browse_id)), + job_mode: { + let mode = store.get_sandbox_job_mode(job.id).await.ok().flatten(); + mode.filter(|m| m != "worker") + }, + transitions, + })); } - Ok(Json(JobInfo { - id: ctx.job_id, - title: ctx.title.clone(), - state: ctx.state.to_string(), - user_id: ctx.user_id.clone(), - created_at: ctx.created_at.to_rfc3339(), - started_at: ctx.started_at.map(|dt| dt.to_rfc3339()), - })) + Err((StatusCode::NOT_FOUND, "Job not found".to_string())) } async fn jobs_cancel_handler( State(state): State>, Path(id): Path, ) -> Result, (StatusCode, String)> { - let context_manager = state.context_manager.as_ref().ok_or(( + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Try sandbox job cancellation, scoped to the authenticated user. + if let Some(ref store) = state.store + && let Ok(Some(job)) = store.get_sandbox_job(job_id).await + { + if job.user_id != state.user_id { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + if job.status == "running" || job.status == "creating" { + // Stop the container if we have a job manager. + if let Some(ref jm) = state.job_manager + && let Err(e) = jm.stop_job(job_id).await + { + tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation"); + } + store + .update_sandbox_job_status( + job_id, + "failed", + Some(false), + Some("Cancelled by user"), + None, + Some(chrono::Utc::now()), + ) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + } + return Ok(Json(serde_json::json!({ + "status": "cancelled", + "job_id": job_id, + }))); + } + + Err((StatusCode::NOT_FOUND, "Job not found".to_string())) +} + +async fn jobs_restart_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( StatusCode::SERVICE_UNAVAILABLE, - "Context manager not available".to_string(), + "Database not available".to_string(), + ))?; + let jm = state.job_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Sandbox not enabled".to_string(), + ))?; + + let old_job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let old_job = store + .get_sandbox_job(old_job_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; + + // Scope to the authenticated user. + if old_job.user_id != state.user_id { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + + if old_job.status != "interrupted" && old_job.status != "failed" { + return Err(( + StatusCode::CONFLICT, + format!("Cannot restart job in state '{}'", old_job.status), + )); + } + + // Create a new job with the same task and project_dir. + let new_job_id = Uuid::new_v4(); + let now = chrono::Utc::now(); + + let record = crate::history::SandboxJobRecord { + id: new_job_id, + task: old_job.task.clone(), + status: "creating".to_string(), + user_id: old_job.user_id.clone(), + project_dir: old_job.project_dir.clone(), + success: None, + failure_reason: None, + created_at: now, + started_at: None, + completed_at: None, + }; + store + .save_sandbox_job(&record) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + // Look up the original job's mode so the restart uses the same mode. + let mode = match store.get_sandbox_job_mode(old_job_id).await { + Ok(Some(m)) if m == "claude_code" => crate::orchestrator::job_manager::JobMode::ClaudeCode, + _ => crate::orchestrator::job_manager::JobMode::Worker, + }; + + let project_dir = std::path::PathBuf::from(&old_job.project_dir); + let _token = jm + .create_job(new_job_id, &old_job.task, Some(project_dir), mode) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create container: {}", e), + ) + })?; + + store + .update_sandbox_job_status(new_job_id, "running", None, None, Some(now), None) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + Ok(Json(serde_json::json!({ + "status": "restarted", + "old_job_id": old_job_id, + "new_job_id": new_job_id, + }))) +} + +// --- Claude Code prompt and events handlers --- + +/// Submit a follow-up prompt to a running Claude Code sandbox job. +async fn jobs_prompt_handler( + State(state): State>, + Path(id): Path, + Json(body): Json, +) -> Result, (StatusCode, String)> { + let prompt_queue = state.prompt_queue.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Claude Code not configured".to_string(), + ))?; + + let job_id: uuid::Uuid = id + .parse() + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Verify user owns this job. + if let Some(ref store) = state.store + && !store + .sandbox_job_belongs_to_user(job_id, &state.user_id) + .await + .unwrap_or(false) + { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + + let content = body + .get("content") + .and_then(|v| v.as_str()) + .ok_or(( + StatusCode::BAD_REQUEST, + "Missing 'content' field".to_string(), + ))? + .to_string(); + + let done = body.get("done").and_then(|v| v.as_bool()).unwrap_or(false); + + let prompt = crate::orchestrator::api::PendingPrompt { content, done }; + + { + let mut queue = prompt_queue.lock().await; + queue.entry(job_id).or_default().push_back(prompt); + } + + Ok(Json(serde_json::json!({ + "status": "queued", + "job_id": job_id.to_string(), + }))) +} + +/// Load persisted job events for a job (for history replay on page open). +async fn jobs_events_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Database not available".to_string(), + ))?; + + let job_id: uuid::Uuid = id + .parse() + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Verify user owns this job. + if !store + .sandbox_job_belongs_to_user(job_id, &state.user_id) + .await + .unwrap_or(false) + { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + + let events = store + .list_job_events(job_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let events_json: Vec = events + .into_iter() + .map(|e| { + serde_json::json!({ + "id": e.id, + "event_type": e.event_type, + "data": e.data, + "created_at": e.created_at.to_rfc3339(), + }) + }) + .collect(); + + Ok(Json(serde_json::json!({ + "job_id": job_id.to_string(), + "events": events_json, + }))) +} + +// --- Project file handlers for sandbox jobs --- + +#[derive(Deserialize)] +struct FilePathQuery { + path: Option, +} + +async fn job_files_list_handler( + State(state): State>, + Path(id): Path, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), ))?; let job_id = Uuid::parse_str(&id) .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - let ctx = context_manager - .get_context(job_id) + let job = store + .get_sandbox_job(job_id) .await - .map_err(|_| (StatusCode::NOT_FOUND, "Job not found".to_string()))?; + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; - if ctx.user_id != state.user_id { + // Verify user owns this job. + if job.user_id != state.user_id { return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); } - context_manager - .update_context(job_id, |ctx| { - ctx.transition_to(crate::context::JobState::Cancelled, None) - }) + let base = std::path::PathBuf::from(&job.project_dir); + let rel_path = query.path.as_deref().unwrap_or(""); + let target = base.join(rel_path); + + // Path traversal guard. + let canonical = target + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Path not found".to_string()))?; + let base_canonical = base + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; + if !canonical.starts_with(&base_canonical) { + return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); + } + + let mut entries = Vec::new(); + let mut read_dir = tokio::fs::read_dir(&canonical) + .await + .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read directory".to_string()))?; + + while let Ok(Some(entry)) = read_dir.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + let is_dir = entry + .file_type() + .await + .map(|ft| ft.is_dir()) + .unwrap_or(false); + let rel = if rel_path.is_empty() { + name.clone() + } else { + format!("{}/{}", rel_path, name) + }; + entries.push(ProjectFileEntry { + name, + path: rel, + is_dir, + }); + } + + entries.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.cmp(&b.name))); + + Ok(Json(ProjectFilesResponse { entries })) +} + +async fn job_files_read_handler( + State(state): State>, + Path(id): Path, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let job = store + .get_sandbox_job(job_id) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .map_err(|msg| (StatusCode::CONFLICT, msg))?; + .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; - Ok(Json(serde_json::json!({ - "status": "cancelled", - "job_id": job_id, - }))) + // Verify user owns this job. + if job.user_id != state.user_id { + return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); + } + + let path = query.path.as_deref().ok_or(( + StatusCode::BAD_REQUEST, + "path parameter required".to_string(), + ))?; + + let base = std::path::PathBuf::from(&job.project_dir); + let file_path = base.join(path); + + let canonical = file_path + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "File not found".to_string()))?; + let base_canonical = base + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; + if !canonical.starts_with(&base_canonical) { + return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); + } + + let content = tokio::fs::read_to_string(&canonical) + .await + .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read file".to_string()))?; + + Ok(Json(ProjectFileReadResponse { + path: path.to_string(), + content, + })) } // --- Logs handlers --- @@ -841,6 +1707,70 @@ async fn extensions_activate_handler( } } +// --- Project file serving handlers --- + +/// Redirect `/projects/{id}` to `/projects/{id}/` so relative paths in +/// the served HTML resolve within the project namespace. +async fn project_redirect_handler(Path(project_id): Path) -> impl IntoResponse { + axum::response::Redirect::permanent(&format!("/projects/{project_id}/")) +} + +/// Serve `index.html` when hitting `/projects/{project_id}/`. +async fn project_index_handler(Path(project_id): Path) -> impl IntoResponse { + serve_project_file(&project_id, "index.html").await +} + +/// Serve any file under `/projects/{project_id}/{path}`. +async fn project_file_handler( + Path((project_id, path)): Path<(String, String)>, +) -> impl IntoResponse { + serve_project_file(&project_id, &path).await +} + +/// Shared logic: resolve the file inside `~/.ironclaw/projects/{project_id}/`, +/// guard against path traversal, and stream the content with the right MIME type. +async fn serve_project_file(project_id: &str, path: &str) -> axum::response::Response { + // Reject project_id values that could escape the projects directory. + if project_id.contains('/') + || project_id.contains('\\') + || project_id.contains("..") + || project_id.is_empty() + { + return (StatusCode::BAD_REQUEST, "Invalid project ID").into_response(); + } + + let base = dirs::home_dir() + .unwrap_or_else(|| std::path::PathBuf::from(".")) + .join(".ironclaw") + .join("projects") + .join(project_id); + + let file_path = base.join(path); + + // Path traversal guard + let canonical = match file_path.canonicalize() { + Ok(p) => p, + Err(_) => return (StatusCode::NOT_FOUND, "Not found").into_response(), + }; + let base_canonical = match base.canonicalize() { + Ok(p) => p, + Err(_) => return (StatusCode::NOT_FOUND, "Not found").into_response(), + }; + if !canonical.starts_with(&base_canonical) { + return (StatusCode::FORBIDDEN, "Forbidden").into_response(); + } + + match tokio::fs::read(&canonical).await { + Ok(contents) => { + let mime = mime_guess::from_path(&canonical) + .first_or_octet_stream() + .to_string(); + ([(header::CONTENT_TYPE, mime)], contents).into_response() + } + Err(_) => (StatusCode::NOT_FOUND, "Not found").into_response(), + } +} + async fn extensions_remove_handler( State(state): State>, Path(name): Path, @@ -856,6 +1786,446 @@ async fn extensions_remove_handler( } } +// --- Routines handlers --- + +async fn routines_list_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routines = store + .list_routines(&state.user_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let items: Vec = routines.iter().map(routine_to_info).collect(); + + Ok(Json(RoutineListResponse { routines: items })) +} + +async fn routines_summary_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routines = store + .list_routines(&state.user_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let total = routines.len() as u64; + let enabled = routines.iter().filter(|r| r.enabled).count() as u64; + let disabled = total - enabled; + let failing = routines + .iter() + .filter(|r| r.consecutive_failures > 0) + .count() as u64; + + let today_start = chrono::Utc::now() + .date_naive() + .and_hms_opt(0, 0, 0) + .map(|dt| dt.and_utc()); + let runs_today = if let Some(start) = today_start { + routines + .iter() + .filter(|r| r.last_run_at.is_some_and(|ts| ts >= start)) + .count() as u64 + } else { + 0 + }; + + Ok(Json(RoutineSummaryResponse { + total, + enabled, + disabled, + failing, + runs_today, + })) +} + +async fn routines_detail_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let routine = store + .get_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; + + let runs = store + .list_routine_runs(routine_id, 20) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let recent_runs: Vec = runs + .iter() + .map(|run| RoutineRunInfo { + id: run.id, + trigger_type: run.trigger_type.clone(), + started_at: run.started_at.to_rfc3339(), + completed_at: run.completed_at.map(|dt| dt.to_rfc3339()), + status: format!("{:?}", run.status), + result_summary: run.result_summary.clone(), + tokens_used: run.tokens_used, + }) + .collect(); + + Ok(Json(RoutineDetailResponse { + id: routine.id, + name: routine.name.clone(), + description: routine.description.clone(), + enabled: routine.enabled, + trigger: serde_json::to_value(&routine.trigger).unwrap_or_default(), + action: serde_json::to_value(&routine.action).unwrap_or_default(), + guardrails: serde_json::to_value(&routine.guardrails).unwrap_or_default(), + notify: serde_json::to_value(&routine.notify).unwrap_or_default(), + last_run_at: routine.last_run_at.map(|dt| dt.to_rfc3339()), + next_fire_at: routine.next_fire_at.map(|dt| dt.to_rfc3339()), + run_count: routine.run_count, + consecutive_failures: routine.consecutive_failures, + created_at: routine.created_at.to_rfc3339(), + recent_runs, + })) +} + +async fn routines_trigger_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let routine = store + .get_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; + + // Send the routine prompt through the message pipeline as a manual trigger. + let prompt = match &routine.action { + crate::agent::routine::RoutineAction::Lightweight { prompt, .. } => prompt.clone(), + crate::agent::routine::RoutineAction::FullJob { + title, description, .. + } => format!("{}: {}", title, description), + }; + + let content = format!("[routine:{}] {}", routine.name, prompt); + let msg = IncomingMessage::new("gateway", &state.user_id, content); + + let tx_guard = state.msg_tx.read().await; + let tx = tx_guard.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Channel not started".to_string(), + ))?; + + tx.send(msg).await.map_err(|_| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Channel closed".to_string(), + ) + })?; + + Ok(Json(serde_json::json!({ + "status": "triggered", + "routine_id": routine_id, + }))) +} + +#[derive(Deserialize)] +struct ToggleRequest { + enabled: Option, +} + +async fn routines_toggle_handler( + State(state): State>, + Path(id): Path, + body: Option>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let mut routine = store + .get_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; + + // If a specific value was provided, use it; otherwise toggle. + routine.enabled = match body { + Some(Json(req)) => req.enabled.unwrap_or(!routine.enabled), + None => !routine.enabled, + }; + + store + .update_routine(&routine) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + Ok(Json(serde_json::json!({ + "status": if routine.enabled { "enabled" } else { "disabled" }, + "routine_id": routine_id, + }))) +} + +async fn routines_delete_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let deleted = store + .delete_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if deleted { + Ok(Json(serde_json::json!({ + "status": "deleted", + "routine_id": routine_id, + }))) + } else { + Err((StatusCode::NOT_FOUND, "Routine not found".to_string())) + } +} + +async fn routines_runs_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let runs = store + .list_routine_runs(routine_id, 50) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let run_infos: Vec = runs + .iter() + .map(|run| RoutineRunInfo { + id: run.id, + trigger_type: run.trigger_type.clone(), + started_at: run.started_at.to_rfc3339(), + completed_at: run.completed_at.map(|dt| dt.to_rfc3339()), + status: format!("{:?}", run.status), + result_summary: run.result_summary.clone(), + tokens_used: run.tokens_used, + }) + .collect(); + + Ok(Json(serde_json::json!({ + "routine_id": routine_id, + "runs": run_infos, + }))) +} + +/// Convert a Routine to the trimmed RoutineInfo for list display. +fn routine_to_info(r: &crate::agent::routine::Routine) -> RoutineInfo { + let (trigger_type, trigger_summary) = match &r.trigger { + crate::agent::routine::Trigger::Cron { schedule } => { + ("cron".to_string(), format!("cron: {}", schedule)) + } + crate::agent::routine::Trigger::Event { + pattern, channel, .. + } => { + let ch = channel.as_deref().unwrap_or("any"); + ("event".to_string(), format!("on {} /{}/", ch, pattern)) + } + crate::agent::routine::Trigger::Webhook { path, .. } => { + let p = path.as_deref().unwrap_or("/"); + ("webhook".to_string(), format!("webhook: {}", p)) + } + crate::agent::routine::Trigger::Manual => ("manual".to_string(), "manual only".to_string()), + }; + + let action_type = match &r.action { + crate::agent::routine::RoutineAction::Lightweight { .. } => "lightweight", + crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", + }; + + let status = if !r.enabled { + "disabled" + } else if r.consecutive_failures > 0 { + "failing" + } else { + "active" + }; + + RoutineInfo { + id: r.id, + name: r.name.clone(), + description: r.description.clone(), + enabled: r.enabled, + trigger_type, + trigger_summary, + action_type: action_type.to_string(), + last_run_at: r.last_run_at.map(|dt| dt.to_rfc3339()), + next_fire_at: r.next_fire_at.map(|dt| dt.to_rfc3339()), + run_count: r.run_count, + consecutive_failures: r.consecutive_failures, + status: status.to_string(), + } +} + +// --- Settings handlers --- + +async fn settings_list_handler( + State(state): State>, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let rows = store.list_settings(&state.user_id).await.map_err(|e| { + tracing::error!("Failed to list settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let settings = rows + .into_iter() + .map(|r| SettingResponse { + key: r.key, + value: r.value, + updated_at: r.updated_at.to_rfc3339(), + }) + .collect(); + + Ok(Json(SettingsListResponse { settings })) +} + +async fn settings_get_handler( + State(state): State>, + Path(key): Path, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let row = store + .get_setting_full(&state.user_id, &key) + .await + .map_err(|e| { + tracing::error!("Failed to get setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or(StatusCode::NOT_FOUND)?; + + Ok(Json(SettingResponse { + key: row.key, + value: row.value, + updated_at: row.updated_at.to_rfc3339(), + })) +} + +async fn settings_set_handler( + State(state): State>, + Path(key): Path, + Json(body): Json, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .set_setting(&state.user_id, &key, &body.value) + .await + .map_err(|e| { + tracing::error!("Failed to set setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} + +async fn settings_delete_handler( + State(state): State>, + Path(key): Path, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .delete_setting(&state.user_id, &key) + .await + .map_err(|e| { + tracing::error!("Failed to delete setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} + +async fn settings_export_handler( + State(state): State>, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let settings = store.get_all_settings(&state.user_id).await.map_err(|e| { + tracing::error!("Failed to export settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(Json(SettingsExportResponse { settings })) +} + +async fn settings_import_handler( + State(state): State>, + Json(body): Json, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .set_all_settings(&state.user_id, &body.settings) + .await + .map_err(|e| { + tracing::error!("Failed to import settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} + // --- Gateway control plane handlers --- async fn gateway_status_handler( @@ -881,3 +2251,84 @@ struct GatewayStatusResponse { ws_connections: u64, total_connections: u64, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_turns_from_db_messages_complete() { + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Hi there!".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "How are you?".to_string(), + created_at: now + chrono::TimeDelta::seconds(2), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Doing well!".to_string(), + created_at: now + chrono::TimeDelta::seconds(3), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 2); + assert_eq!(turns[0].user_input, "Hello"); + assert_eq!(turns[0].response.as_deref(), Some("Hi there!")); + assert_eq!(turns[0].state, "Completed"); + assert_eq!(turns[1].user_input, "How are you?"); + assert_eq!(turns[1].response.as_deref(), Some("Doing well!")); + } + + #[test] + fn test_build_turns_from_db_messages_incomplete_last() { + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Hi!".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Lost message".to_string(), + created_at: now + chrono::TimeDelta::seconds(2), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 2); + assert_eq!(turns[1].user_input, "Lost message"); + assert!(turns[1].response.is_none()); + assert_eq!(turns[1].state, "Failed"); + } + + #[test] + fn test_build_turns_from_db_messages_empty() { + let turns = build_turns_from_db_messages(&[]); + assert!(turns.is_empty()); + } +} diff --git a/src/channels/web/sse.rs b/src/channels/web/sse.rs index 240c9c2b..120a7103 100644 --- a/src/channels/web/sse.rs +++ b/src/channels/web/sse.rs @@ -13,10 +13,15 @@ use tokio_stream::wrappers::BroadcastStream; use crate::channels::web::types::SseEvent; +/// Maximum number of concurrent SSE/WebSocket connections. +/// Prevents resource exhaustion from connection flooding. +const MAX_CONNECTIONS: u64 = 100; + /// Manages SSE broadcast to all connected browser tabs. pub struct SseManager { tx: broadcast::Sender, connection_count: Arc, + max_connections: u64, } impl SseManager { @@ -27,6 +32,7 @@ impl SseManager { Self { tx, connection_count: Arc::new(AtomicU64::new(0)), + max_connections: MAX_CONNECTIONS, } } @@ -45,25 +51,50 @@ impl SseManager { /// /// Returns a stream of `SseEvent` values and increments/decrements the /// connection counter on creation/drop, just like `subscribe()` does for SSE. - pub fn subscribe_raw(&self) -> impl Stream + Send + 'static + use<> { + /// + /// Returns `None` if the maximum connection limit has been reached. + pub fn subscribe_raw(&self) -> Option + Send + 'static + use<>> { + // Atomically increment only if below the limit. This prevents + // concurrent callers from overshooting max_connections. let counter = Arc::clone(&self.connection_count); - counter.fetch_add(1, Ordering::Relaxed); + let max = self.max_connections; + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + if current < max { + Some(current + 1) + } else { + None + } + }) + .ok()?; let rx = self.tx.subscribe(); let stream = BroadcastStream::new(rx).filter_map(|result| result.ok()); - CountedStream { + Some(CountedStream { inner: stream, counter, - } + }) } /// Create a new SSE stream for a client connection. + /// + /// Returns `None` if the maximum connection limit has been reached. pub fn subscribe( &self, - ) -> Sse> + Send + 'static + use<>> { + ) -> Option> + Send + 'static + use<>>> { + // Atomically increment only if below the limit. let counter = Arc::clone(&self.connection_count); - counter.fetch_add(1, Ordering::Relaxed); + let max = self.max_connections; + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + if current < max { + Some(current + 1) + } else { + None + } + }) + .ok()?; let rx = self.tx.subscribe(); let stream = BroadcastStream::new(rx) @@ -79,7 +110,15 @@ impl SseManager { SseEvent::StreamChunk { .. } => "stream_chunk", SseEvent::Status { .. } => "status", SseEvent::ApprovalNeeded { .. } => "approval_needed", + SseEvent::AuthRequired { .. } => "auth_required", + SseEvent::AuthCompleted { .. } => "auth_completed", SseEvent::Error { .. } => "error", + SseEvent::JobStarted { .. } => "job_started", + SseEvent::JobMessage { .. } => "job_message", + SseEvent::JobToolUse { .. } => "job_tool_use", + SseEvent::JobToolResult { .. } => "job_tool_result", + SseEvent::JobStatus { .. } => "job_status", + SseEvent::JobResult { .. } => "job_result", SseEvent::Heartbeat => "heartbeat", }; Ok(Event::default().event(event_type).data(data)) @@ -91,8 +130,10 @@ impl SseManager { counter, }; - Sse::new(counted_stream) - .keep_alive(KeepAlive::new().interval(Duration::from_secs(30)).text("")) + Some( + Sse::new(counted_stream) + .keep_alive(KeepAlive::new().interval(Duration::from_secs(30)).text("")), + ) } } @@ -152,13 +193,14 @@ mod tests { manager.broadcast(SseEvent::Status { message: "test".to_string(), + thread_id: None, }); let event = rx.next().await; assert!(event.is_some()); let event = event.unwrap().unwrap(); match event { - SseEvent::Status { message } => assert_eq!(message, "test"), + SseEvent::Status { message, .. } => assert_eq!(message, "test"), _ => panic!("unexpected event type"), } } @@ -166,17 +208,18 @@ mod tests { #[tokio::test] async fn test_subscribe_raw_receives_events() { let manager = SseManager::new(); - let mut stream = Box::pin(manager.subscribe_raw()); + let mut stream = Box::pin(manager.subscribe_raw().expect("should subscribe")); assert_eq!(manager.connection_count(), 1); manager.broadcast(SseEvent::Thinking { message: "working".to_string(), + thread_id: None, }); let event = stream.next().await.unwrap(); match event { - SseEvent::Thinking { message } => assert_eq!(message, "working"), + SseEvent::Thinking { message, .. } => assert_eq!(message, "working"), _ => panic!("Expected Thinking event"), } } @@ -185,7 +228,7 @@ mod tests { async fn test_subscribe_raw_decrements_on_drop() { let manager = SseManager::new(); { - let _stream = Box::pin(manager.subscribe_raw()); + let _stream = Box::pin(manager.subscribe_raw().expect("should subscribe")); assert_eq!(manager.connection_count(), 1); } // Stream dropped, counter should decrement @@ -195,8 +238,8 @@ mod tests { #[tokio::test] async fn test_subscribe_raw_multiple_subscribers() { let manager = SseManager::new(); - let mut s1 = Box::pin(manager.subscribe_raw()); - let mut s2 = Box::pin(manager.subscribe_raw()); + let mut s1 = Box::pin(manager.subscribe_raw().expect("should subscribe")); + let mut s2 = Box::pin(manager.subscribe_raw().expect("should subscribe")); assert_eq!(manager.connection_count(), 2); manager.broadcast(SseEvent::Heartbeat); @@ -211,4 +254,18 @@ mod tests { drop(s2); assert_eq!(manager.connection_count(), 0); } + + #[tokio::test] + async fn test_subscribe_raw_rejects_over_limit() { + let mut manager = SseManager::new(); + manager.max_connections = 2; // Low limit for testing + + let _s1 = Box::pin(manager.subscribe_raw().expect("first should succeed")); + let _s2 = Box::pin(manager.subscribe_raw().expect("second should succeed")); + assert_eq!(manager.connection_count(), 2); + + // Third should be rejected + assert!(manager.subscribe_raw().is_none()); + assert!(manager.subscribe().is_none()); + } } diff --git a/src/channels/web/static/app.js b/src/channels/web/static/app.js index a7ac57dd..cdc1e06d 100644 --- a/src/channels/web/static/app.js +++ b/src/channels/web/static/app.js @@ -4,6 +4,14 @@ let token = ''; let eventSource = null; let logEventSource = null; let currentTab = 'chat'; +let currentThreadId = null; +let assistantThreadId = null; +let hasMore = false; +let oldestTimestamp = null; +let loadingOlder = false; +let jobEvents = new Map(); // job_id -> Array of events +let jobListRefreshTimer = null; +const JOB_EVENTS_CAP = 500; // --- Auth --- @@ -17,15 +25,24 @@ function authenticate() { // Test the token against the health-ish endpoint (chat/threads requires auth) apiFetch('/api/chat/threads') .then(() => { + sessionStorage.setItem('ironclaw_token', token); document.getElementById('auth-screen').style.display = 'none'; document.getElementById('app').style.display = 'flex'; + // Strip token from URL so it's not visible in the address bar + const cleaned = new URL(window.location); + cleaned.searchParams.delete('token'); + window.history.replaceState({}, '', cleaned.pathname + cleaned.search); connectSSE(); connectLogSSE(); - loadHistory(); + startGatewayStatusPolling(); + loadThreads(); loadMemoryTree(); loadJobs(); }) .catch(() => { + sessionStorage.removeItem('ironclaw_token'); + document.getElementById('auth-screen').style.display = ''; + document.getElementById('app').style.display = 'none'; document.getElementById('auth-error').textContent = 'Invalid token'; }); } @@ -34,6 +51,26 @@ document.getElementById('token-input').addEventListener('keydown', (e) => { if (e.key === 'Enter') authenticate(); }); +// Auto-authenticate from URL param or saved session +(function autoAuth() { + const params = new URLSearchParams(window.location.search); + const urlToken = params.get('token'); + if (urlToken) { + document.getElementById('token-input').value = urlToken; + authenticate(); + return; + } + const saved = sessionStorage.getItem('ironclaw_token'); + if (saved) { + document.getElementById('token-input').value = saved; + // Hide auth screen immediately to prevent flash, authenticate() will + // restore it if the token turns out to be invalid. + document.getElementById('auth-screen').style.display = 'none'; + document.getElementById('app').style.display = 'flex'; + authenticate(); + } +})(); + // --- API helper --- function apiFetch(path, options) { @@ -69,34 +106,54 @@ function connectSSE() { eventSource.addEventListener('response', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; addMessage('assistant', data.content); setStatus(''); + enableChatInput(); + // Refresh thread list so new titles appear after first message + loadThreads(); }); eventSource.addEventListener('thinking', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; setStatus(data.message, true); }); eventSource.addEventListener('tool_started', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; setStatus('Running tool: ' + data.name, true); }); eventSource.addEventListener('tool_completed', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; const icon = data.success ? '\u2713' : '\u2717'; setStatus('Tool ' + data.name + ' ' + icon); }); eventSource.addEventListener('stream_chunk', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; appendToLastAssistant(data.content); }); eventSource.addEventListener('status', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; setStatus(data.message); + // "Done" and "Awaiting approval" are terminal signals from the agent: + // the agentic loop finished, so re-enable input as a safety net in case + // the response SSE event is empty or lost. + if (data.message === 'Done' || data.message === 'Awaiting approval') { + enableChatInput(); + } + }); + + eventSource.addEventListener('job_started', (e) => { + const data = JSON.parse(e.data); + showJobCard(data); }); eventSource.addEventListener('approval_needed', (e) => { @@ -104,18 +161,70 @@ function connectSSE() { showApproval(data); }); + eventSource.addEventListener('auth_required', (e) => { + const data = JSON.parse(e.data); + showAuthCard(data); + }); + + eventSource.addEventListener('auth_completed', (e) => { + const data = JSON.parse(e.data); + removeAuthCard(data.extension_name); + showToast(data.message, 'success'); + enableChatInput(); + }); + eventSource.addEventListener('error', (e) => { if (e.data) { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; addMessage('system', 'Error: ' + data.message); + enableChatInput(); } }); + + // Job event listeners (activity stream for all sandbox jobs) + const jobEventTypes = [ + 'job_message', 'job_tool_use', 'job_tool_result', + 'job_status', 'job_result' + ]; + for (const evtType of jobEventTypes) { + eventSource.addEventListener(evtType, (e) => { + const data = JSON.parse(e.data); + const jobId = data.job_id; + if (!jobId) return; + if (!jobEvents.has(jobId)) jobEvents.set(jobId, []); + const events = jobEvents.get(jobId); + events.push({ type: evtType, data: data, ts: Date.now() }); + // Cap per-job events to prevent memory leak + while (events.length > JOB_EVENTS_CAP) events.shift(); + // If the Activity tab is currently visible for this job, refresh it + refreshActivityTab(jobId); + // Auto-refresh job list when on jobs tab (debounced) + if ((evtType === 'job_result' || evtType === 'job_status') && currentTab === 'jobs' && !currentJobId) { + clearTimeout(jobListRefreshTimer); + jobListRefreshTimer = setTimeout(loadJobs, 200); + } + // Clean up finished job events after a viewing window + if (evtType === 'job_result') { + setTimeout(() => jobEvents.delete(jobId), 60000); + } + }); + } +} + +// Check if an SSE event belongs to the currently viewed thread. +// Events without a thread_id (legacy) are always shown. +function isCurrentThread(threadId) { + if (!threadId) return true; + if (!currentThreadId) return true; + return threadId === currentThreadId; } // --- Chat --- function sendMessage() { const input = document.getElementById('chat-input'); + const sendBtn = document.getElementById('send-btn'); const content = input.value.trim(); if (!content) return; @@ -124,19 +233,31 @@ function sendMessage() { autoResizeTextarea(input); setStatus('Sending...', true); + sendBtn.disabled = true; + input.disabled = true; + apiFetch('/api/chat/send', { method: 'POST', - body: { content }, + body: { content, thread_id: currentThreadId || undefined }, }).catch((err) => { addMessage('system', 'Failed to send: ' + err.message); setStatus(''); + enableChatInput(); }); } +function enableChatInput() { + const input = document.getElementById('chat-input'); + const sendBtn = document.getElementById('send-btn'); + sendBtn.disabled = false; + input.disabled = false; + input.focus(); +} + function sendApprovalAction(requestId, action) { apiFetch('/api/chat/approval', { method: 'POST', - body: { request_id: requestId, action: action }, + body: { request_id: requestId, action: action, thread_id: currentThreadId }, }).catch((err) => { addMessage('system', 'Failed to send approval: ' + err.message); }); @@ -159,11 +280,48 @@ function sendApprovalAction(requestId, action) { function renderMarkdown(text) { if (typeof marked !== 'undefined') { - return marked.parse(text); + let html = marked.parse(text); + // Sanitize HTML output to prevent XSS from tool output or LLM responses. + html = sanitizeRenderedHtml(html); + // Inject copy buttons into
 blocks
+    html = html.replace(/
/g, '
');
+    return html;
   }
   return escapeHtml(text);
 }
 
+// Strip dangerous HTML elements and attributes from rendered markdown.
+// This prevents XSS from tool output or prompt injection in LLM responses.
+function sanitizeRenderedHtml(html) {
+  html = html.replace(/)<[^<]*)*<\/script>/gi, '');
+  html = html.replace(/]*>[\s\S]*?<\/iframe>/gi, '');
+  html = html.replace(/]*>[\s\S]*?<\/object>/gi, '');
+  html = html.replace(/]*\/?>/gi, '');
+  html = html.replace(/]*>[\s\S]*?<\/form>/gi, '');
+  html = html.replace(/]*>[\s\S]*?<\/style>/gi, '');
+  html = html.replace(/]*\/?>/gi, '');
+  html = html.replace(/]*\/?>/gi, '');
+  html = html.replace(/]*\/?>/gi, '');
+  // Remove event handler attributes (onclick, onerror, onload, etc.)
+  html = html.replace(/\s+on\w+\s*=\s*"[^"]*"/gi, '');
+  html = html.replace(/\s+on\w+\s*=\s*'[^']*'/gi, '');
+  html = html.replace(/\s+on\w+\s*=\s*[^\s>]+/gi, '');
+  // Remove javascript: and data: URLs in href/src attributes
+  html = html.replace(/(href|src|action)\s*=\s*["']?\s*javascript\s*:/gi, '$1="');
+  html = html.replace(/(href|src|action)\s*=\s*["']?\s*data\s*:/gi, '$1="');
+  return html;
+}
+
+function copyCodeBlock(btn) {
+  const pre = btn.parentElement;
+  const code = pre.querySelector('code');
+  const text = code ? code.textContent : pre.textContent;
+  navigator.clipboard.writeText(text).then(() => {
+    btn.textContent = 'Copied!';
+    setTimeout(() => { btn.textContent = 'Copy'; }, 1500);
+  });
+}
+
 function addMessage(role, content) {
   const container = document.getElementById('chat-messages');
   const div = document.createElement('div');
@@ -268,19 +426,340 @@ function showApproval(data) {
   container.scrollTop = container.scrollHeight;
 }
 
-function loadHistory() {
-  apiFetch('/api/chat/history').then((data) => {
-    const container = document.getElementById('chat-messages');
-    container.innerHTML = '';
-    for (const turn of data.turns) {
-      addMessage('user', turn.user_input);
-      if (turn.response) {
-        addMessage('assistant', turn.response);
-      }
-    }
-  }).catch(() => {
-    // No history or no active thread, that's fine
+function showJobCard(data) {
+  const container = document.getElementById('chat-messages');
+  const card = document.createElement('div');
+  card.className = 'job-card';
+
+  const icon = document.createElement('span');
+  icon.className = 'job-card-icon';
+  icon.textContent = '\u2692';
+  card.appendChild(icon);
+
+  const info = document.createElement('div');
+  info.className = 'job-card-info';
+
+  const title = document.createElement('div');
+  title.className = 'job-card-title';
+  title.textContent = data.title || 'Sandbox Job';
+  info.appendChild(title);
+
+  const id = document.createElement('div');
+  id.className = 'job-card-id';
+  id.textContent = (data.job_id || '').substring(0, 8);
+  info.appendChild(id);
+
+  card.appendChild(info);
+
+  const viewBtn = document.createElement('button');
+  viewBtn.className = 'job-card-view';
+  viewBtn.textContent = 'View Job';
+  viewBtn.addEventListener('click', () => {
+    switchTab('jobs');
+    openJobDetail(data.job_id);
   });
+  card.appendChild(viewBtn);
+
+  if (data.browse_url) {
+    const browseBtn = document.createElement('a');
+    browseBtn.className = 'job-card-browse';
+    browseBtn.href = data.browse_url;
+    browseBtn.target = '_blank';
+    browseBtn.textContent = 'Browse';
+    card.appendChild(browseBtn);
+  }
+
+  container.appendChild(card);
+  container.scrollTop = container.scrollHeight;
+}
+
+// --- Auth card ---
+
+function showAuthCard(data) {
+  // Remove any existing card for this extension first
+  removeAuthCard(data.extension_name);
+
+  const container = document.getElementById('chat-messages');
+  const card = document.createElement('div');
+  card.className = 'auth-card';
+  card.setAttribute('data-extension-name', data.extension_name);
+
+  const header = document.createElement('div');
+  header.className = 'auth-header';
+  header.textContent = 'Authentication required for ' + data.extension_name;
+  card.appendChild(header);
+
+  if (data.instructions) {
+    const instr = document.createElement('div');
+    instr.className = 'auth-instructions';
+    instr.textContent = data.instructions;
+    card.appendChild(instr);
+  }
+
+  const links = document.createElement('div');
+  links.className = 'auth-links';
+
+  if (data.auth_url) {
+    const oauthBtn = document.createElement('button');
+    oauthBtn.className = 'auth-oauth';
+    oauthBtn.textContent = 'Authenticate with ' + data.extension_name;
+    oauthBtn.addEventListener('click', () => {
+      window.open(data.auth_url, '_blank', 'width=600,height=700');
+    });
+    links.appendChild(oauthBtn);
+  }
+
+  if (data.setup_url) {
+    const setupLink = document.createElement('a');
+    setupLink.href = data.setup_url;
+    setupLink.target = '_blank';
+    setupLink.textContent = 'Get your token';
+    links.appendChild(setupLink);
+  }
+
+  if (links.children.length > 0) {
+    card.appendChild(links);
+  }
+
+  // Token input
+  const tokenRow = document.createElement('div');
+  tokenRow.className = 'auth-token-input';
+
+  const tokenInput = document.createElement('input');
+  tokenInput.type = 'password';
+  tokenInput.placeholder = 'Paste your API key or token';
+  tokenInput.addEventListener('keydown', (e) => {
+    if (e.key === 'Enter') submitAuthToken(data.extension_name, tokenInput.value);
+  });
+  tokenRow.appendChild(tokenInput);
+  card.appendChild(tokenRow);
+
+  // Error display (hidden initially)
+  const errorEl = document.createElement('div');
+  errorEl.className = 'auth-error';
+  errorEl.style.display = 'none';
+  card.appendChild(errorEl);
+
+  // Action buttons
+  const actions = document.createElement('div');
+  actions.className = 'auth-actions';
+
+  const submitBtn = document.createElement('button');
+  submitBtn.className = 'auth-submit';
+  submitBtn.textContent = 'Submit';
+  submitBtn.addEventListener('click', () => submitAuthToken(data.extension_name, tokenInput.value));
+
+  const cancelBtn = document.createElement('button');
+  cancelBtn.className = 'auth-cancel';
+  cancelBtn.textContent = 'Cancel';
+  cancelBtn.addEventListener('click', () => cancelAuth(data.extension_name));
+
+  actions.appendChild(submitBtn);
+  actions.appendChild(cancelBtn);
+  card.appendChild(actions);
+
+  container.appendChild(card);
+  container.scrollTop = container.scrollHeight;
+  tokenInput.focus();
+}
+
+function removeAuthCard(extensionName) {
+  const card = document.querySelector('.auth-card[data-extension-name="' + extensionName + '"]');
+  if (card) card.remove();
+}
+
+function submitAuthToken(extensionName, tokenValue) {
+  if (!tokenValue || !tokenValue.trim()) return;
+
+  // Disable submit button while in flight
+  const card = document.querySelector('.auth-card[data-extension-name="' + extensionName + '"]');
+  if (card) {
+    const btns = card.querySelectorAll('button');
+    btns.forEach((b) => { b.disabled = true; });
+  }
+
+  apiFetch('/api/chat/auth-token', {
+    method: 'POST',
+    body: { extension_name: extensionName, token: tokenValue.trim() },
+  }).then((result) => {
+    if (result.success) {
+      removeAuthCard(extensionName);
+      addMessage('system', result.message);
+    } else {
+      showAuthCardError(extensionName, result.message);
+    }
+  }).catch((err) => {
+    showAuthCardError(extensionName, 'Failed: ' + err.message);
+  });
+}
+
+function cancelAuth(extensionName) {
+  apiFetch('/api/chat/auth-cancel', {
+    method: 'POST',
+    body: { extension_name: extensionName },
+  }).catch(() => {});
+  removeAuthCard(extensionName);
+  enableChatInput();
+}
+
+function showAuthCardError(extensionName, message) {
+  const card = document.querySelector('.auth-card[data-extension-name="' + extensionName + '"]');
+  if (!card) return;
+  // Re-enable buttons
+  const btns = card.querySelectorAll('button');
+  btns.forEach((b) => { b.disabled = false; });
+  // Show error
+  const errorEl = card.querySelector('.auth-error');
+  if (errorEl) {
+    errorEl.textContent = message;
+    errorEl.style.display = 'block';
+  }
+}
+
+function loadHistory(before) {
+  let historyUrl = '/api/chat/history?limit=50';
+  if (currentThreadId) {
+    historyUrl += '&thread_id=' + encodeURIComponent(currentThreadId);
+  }
+  if (before) {
+    historyUrl += '&before=' + encodeURIComponent(before);
+  }
+
+  const isPaginating = !!before;
+  if (isPaginating) loadingOlder = true;
+
+  apiFetch(historyUrl).then((data) => {
+    const container = document.getElementById('chat-messages');
+
+    if (!isPaginating) {
+      // Fresh load: clear and render
+      container.innerHTML = '';
+      for (const turn of data.turns) {
+        addMessage('user', turn.user_input);
+        if (turn.response) {
+          addMessage('assistant', turn.response);
+        }
+      }
+    } else {
+      // Pagination: prepend older messages
+      const savedHeight = container.scrollHeight;
+      const fragment = document.createDocumentFragment();
+      for (const turn of data.turns) {
+        const userDiv = createMessageElement('user', turn.user_input);
+        fragment.appendChild(userDiv);
+        if (turn.response) {
+          const assistantDiv = createMessageElement('assistant', turn.response);
+          fragment.appendChild(assistantDiv);
+        }
+      }
+      container.insertBefore(fragment, container.firstChild);
+      // Restore scroll position so the user doesn't jump
+      container.scrollTop = container.scrollHeight - savedHeight;
+    }
+
+    hasMore = data.has_more || false;
+    oldestTimestamp = data.oldest_timestamp || null;
+  }).catch(() => {
+    // No history or no active thread
+  }).finally(() => {
+    loadingOlder = false;
+    removeScrollSpinner();
+  });
+}
+
+// Create a message DOM element without appending it (for prepend operations)
+function createMessageElement(role, content) {
+  const div = document.createElement('div');
+  div.className = 'message ' + role;
+  if (role === 'user') {
+    div.textContent = content;
+  } else {
+    div.setAttribute('data-raw', content);
+    div.innerHTML = renderMarkdown(content);
+  }
+  return div;
+}
+
+function removeScrollSpinner() {
+  const spinner = document.getElementById('scroll-load-spinner');
+  if (spinner) spinner.remove();
+}
+
+// --- Threads ---
+
+function loadThreads() {
+  apiFetch('/api/chat/threads').then((data) => {
+    // Pinned assistant thread
+    if (data.assistant_thread) {
+      assistantThreadId = data.assistant_thread.id;
+      const el = document.getElementById('assistant-thread');
+      const isActive = currentThreadId === assistantThreadId;
+      el.className = 'assistant-item' + (isActive ? ' active' : '');
+      const meta = document.getElementById('assistant-meta');
+      const count = data.assistant_thread.turn_count || 0;
+      meta.textContent = count > 0 ? count + ' turns' : '';
+    }
+
+    // Regular threads
+    const list = document.getElementById('thread-list');
+    list.innerHTML = '';
+    const threads = data.threads || [];
+    for (const thread of threads) {
+      const item = document.createElement('div');
+      item.className = 'thread-item' + (thread.id === currentThreadId ? ' active' : '');
+      const label = document.createElement('span');
+      label.className = 'thread-label';
+      label.textContent = thread.title || thread.id.substring(0, 8);
+      label.title = thread.title ? thread.title + ' (' + thread.id + ')' : thread.id;
+      item.appendChild(label);
+      const meta = document.createElement('span');
+      meta.className = 'thread-meta';
+      meta.textContent = (thread.turn_count || 0) + ' turns';
+      item.appendChild(meta);
+      item.addEventListener('click', () => switchThread(thread.id));
+      list.appendChild(item);
+    }
+
+    // Default to assistant thread on first load if no thread selected
+    if (!currentThreadId && assistantThreadId) {
+      switchToAssistant();
+    }
+  }).catch(() => {});
+}
+
+function switchToAssistant() {
+  if (!assistantThreadId) return;
+  currentThreadId = assistantThreadId;
+  hasMore = false;
+  oldestTimestamp = null;
+  loadHistory();
+  loadThreads();
+}
+
+function switchThread(threadId) {
+  currentThreadId = threadId;
+  hasMore = false;
+  oldestTimestamp = null;
+  loadHistory();
+  loadThreads();
+}
+
+function createNewThread() {
+  apiFetch('/api/chat/thread/new', { method: 'POST' }).then((data) => {
+    currentThreadId = data.id || null;
+    document.getElementById('chat-messages').innerHTML = '';
+    setStatus('');
+    loadThreads();
+  }).catch((err) => {
+    showToast('Failed to create thread: ' + err.message, 'error');
+  });
+}
+
+function toggleThreadSidebar() {
+  const sidebar = document.getElementById('thread-sidebar');
+  sidebar.classList.toggle('collapsed');
+  const btn = document.getElementById('thread-toggle-btn');
+  btn.innerHTML = sidebar.classList.contains('collapsed') ? '»' : '«';
 }
 
 // Chat input auto-resize and keyboard handling
@@ -293,6 +772,20 @@ chatInput.addEventListener('keydown', (e) => {
 });
 chatInput.addEventListener('input', () => autoResizeTextarea(chatInput));
 
+// Infinite scroll: load older messages when scrolled near the top
+document.getElementById('chat-messages').addEventListener('scroll', function () {
+  if (this.scrollTop < 100 && hasMore && !loadingOlder) {
+    loadingOlder = true;
+    // Show spinner at top
+    const spinner = document.createElement('div');
+    spinner.id = 'scroll-load-spinner';
+    spinner.className = 'scroll-load-spinner';
+    spinner.innerHTML = '
Loading older messages...'; + this.insertBefore(spinner, this.firstChild); + loadHistory(oldestTimestamp); + } +}); + function autoResizeTextarea(el) { el.style.height = 'auto'; el.style.height = Math.min(el.scrollHeight, 120) + 'px'; @@ -318,12 +811,16 @@ function switchTab(tab) { if (tab === 'memory') loadMemoryTree(); if (tab === 'jobs') loadJobs(); + if (tab === 'routines') loadRoutines(); + if (tab === 'logs') applyLogFilters(); if (tab === 'extensions') loadExtensions(); } // --- Memory (filesystem tree) --- let memorySearchTimeout = null; +let currentMemoryPath = null; +let currentMemoryContent = null; // Tree state: nested nodes persisted across renders // { name, path, is_dir, children: [] | null, expanded: bool, loaded: bool } let memoryTreeState = null; @@ -437,16 +934,61 @@ function toggleExpand(node) { } function readMemoryFile(path) { + currentMemoryPath = path; // Update breadcrumb - document.getElementById('memory-breadcrumb').innerHTML = buildBreadcrumb(path); + document.getElementById('memory-breadcrumb-path').innerHTML = buildBreadcrumb(path); + document.getElementById('memory-edit-btn').style.display = 'inline-block'; + + // Exit edit mode if active + cancelMemoryEdit(); apiFetch('/api/memory/read?path=' + encodeURIComponent(path)).then((data) => { - document.getElementById('memory-viewer').textContent = data.content; + currentMemoryContent = data.content; + const viewer = document.getElementById('memory-viewer'); + // Render markdown if it's a .md file + if (path.endsWith('.md')) { + viewer.innerHTML = '
' + renderMarkdown(data.content) + '
'; + viewer.classList.add('rendered'); + } else { + viewer.textContent = data.content; + viewer.classList.remove('rendered'); + } }).catch((err) => { + currentMemoryContent = null; document.getElementById('memory-viewer').innerHTML = '
Error: ' + escapeHtml(err.message) + '
'; }); } +function startMemoryEdit() { + if (!currentMemoryPath || currentMemoryContent === null) return; + document.getElementById('memory-viewer').style.display = 'none'; + const editor = document.getElementById('memory-editor'); + editor.style.display = 'flex'; + const textarea = document.getElementById('memory-edit-textarea'); + textarea.value = currentMemoryContent; + textarea.focus(); +} + +function cancelMemoryEdit() { + document.getElementById('memory-viewer').style.display = ''; + document.getElementById('memory-editor').style.display = 'none'; +} + +function saveMemoryEdit() { + if (!currentMemoryPath) return; + const content = document.getElementById('memory-edit-textarea').value; + apiFetch('/api/memory/write', { + method: 'POST', + body: { path: currentMemoryPath, content: content }, + }).then(() => { + showToast('Saved ' + currentMemoryPath, 'success'); + cancelMemoryEdit(); + readMemoryFile(currentMemoryPath); + }).catch((err) => { + showToast('Save failed: ' + err.message, 'error'); + }); +} + function buildBreadcrumb(path) { const parts = path.split('/'); let html = 'workspace'; @@ -472,14 +1014,35 @@ function searchMemory(query) { for (const result of data.results) { const item = document.createElement('div'); item.className = 'search-result'; + const snippet = snippetAround(result.content, query, 120); item.innerHTML = '
' + escapeHtml(result.path) + '
' - + '
' + escapeHtml(result.content.substring(0, 120)) + '
'; + + '
' + highlightQuery(snippet, query) + '
'; item.addEventListener('click', () => readMemoryFile(result.path)); tree.appendChild(item); } }).catch(() => {}); } +function snippetAround(text, query, len) { + const lower = text.toLowerCase(); + const idx = lower.indexOf(query.toLowerCase()); + if (idx < 0) return text.substring(0, len); + const start = Math.max(0, idx - Math.floor(len / 2)); + const end = Math.min(text.length, start + len); + let s = text.substring(start, end); + if (start > 0) s = '...' + s; + if (end < text.length) s = s + '...'; + return s; +} + +function highlightQuery(text, query) { + if (!query) return escapeHtml(text); + const escaped = escapeHtml(text); + const queryEscaped = query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const re = new RegExp('(' + queryEscaped + ')', 'gi'); + return escaped.replace(re, '$1'); +} + // --- Logs --- const LOG_MAX_ENTRIES = 2000; @@ -574,6 +1137,7 @@ function toggleLogsPause() { } function clearLogs() { + if (!confirm('Clear all logs?')) return; document.getElementById('logs-output').innerHTML = ''; logBuffer = []; } @@ -709,39 +1273,53 @@ function activateExtension(name) { } if (res.auth_url) { - addMessage( - 'system', - 'Opening authentication for **' + name + '**. Complete the flow in the opened tab, then click Activate again.' - ); + showToast('Opening authentication for ' + name, 'info'); window.open(res.auth_url, '_blank'); } else if (res.awaiting_token) { - addMessage( - 'system', - (res.instructions || 'Please provide an API token for **' + name + '**.') + - '\n\nYou can authenticate via chat: type `Authenticate ' + name + '` and follow the instructions.' - ); + showToast(res.instructions || 'Please provide an API token for ' + name, 'info'); } else { - addMessage('system', 'Activate failed: ' + res.message); + showToast('Activate failed: ' + res.message, 'error'); } loadExtensions(); }) - .catch((err) => addMessage('system', 'Activate failed: ' + err.message)); + .catch((err) => showToast('Activate failed: ' + err.message, 'error')); } function removeExtension(name) { + if (!confirm('Remove extension "' + name + '"?')) return; apiFetch('/api/extensions/' + encodeURIComponent(name) + '/remove', { method: 'POST' }) .then((res) => { if (!res.success) { - addMessage('system', 'Remove failed: ' + res.message); + showToast('Remove failed: ' + res.message, 'error'); + } else { + showToast('Removed ' + name, 'success'); } loadExtensions(); }) - .catch((err) => addMessage('system', 'Remove failed: ' + err.message)); + .catch((err) => showToast('Remove failed: ' + err.message, 'error')); } // --- Jobs --- +let currentJobId = null; +let currentJobSubTab = 'overview'; +let jobFilesTreeState = null; + function loadJobs() { + currentJobId = null; + jobFilesTreeState = null; + + // Rebuild DOM if renderJobDetail() destroyed it (it wipes .jobs-container innerHTML). + const container = document.querySelector('.jobs-container'); + if (!document.getElementById('jobs-summary')) { + container.innerHTML = + '
' + + '' + + '' + + '
IDTitleStatusCreatedActions
' + + ''; + } + Promise.all([ apiFetch('/api/jobs/summary'), apiFetch('/api/jobs'), @@ -781,27 +1359,792 @@ function renderJobsList(jobs) { tbody.innerHTML = jobs.map((job) => { const shortId = job.id.substring(0, 8); const stateClass = job.state.replace(' ', '_'); - const cancelBtn = (job.state === 'pending' || job.state === 'in_progress') - ? '' - : ''; - return '' + + let actionBtns = ''; + if (job.state === 'pending' || job.state === 'in_progress') { + actionBtns = ''; + } else if (job.state === 'failed' || job.state === 'interrupted') { + actionBtns = ''; + } + + return '' + '' + shortId + '' + '' + escapeHtml(job.title) + '' + '' + escapeHtml(job.state) + '' + '' + formatDate(job.created_at) + '' - + '' + cancelBtn + '' + + '' + actionBtns + '' + ''; }).join(''); } function cancelJob(jobId) { + if (!confirm('Cancel this job?')) return; apiFetch('/api/jobs/' + jobId + '/cancel', { method: 'POST' }) - .then(() => loadJobs()) + .then(() => { + showToast('Job cancelled', 'success'); + if (currentJobId) openJobDetail(currentJobId); + else loadJobs(); + }) .catch((err) => { - addMessage('system', 'Failed to cancel job: ' + err.message); + showToast('Failed to cancel job: ' + err.message, 'error'); }); } +function restartJob(jobId) { + apiFetch('/api/jobs/' + jobId + '/restart', { method: 'POST' }) + .then((res) => { + showToast('Job restarted as ' + (res.new_job_id || '').substring(0, 8), 'success'); + loadJobs(); + }) + .catch((err) => { + showToast('Failed to restart job: ' + err.message, 'error'); + }); +} + +function openJobDetail(jobId) { + currentJobId = jobId; + currentJobSubTab = 'activity'; + apiFetch('/api/jobs/' + jobId).then((job) => { + renderJobDetail(job); + }).catch((err) => { + addMessage('system', 'Failed to load job: ' + err.message); + closeJobDetail(); + }); +} + +function closeJobDetail() { + currentJobId = null; + jobFilesTreeState = null; + loadJobs(); +} + +function renderJobDetail(job) { + const container = document.querySelector('.jobs-container'); + const stateClass = job.state.replace(' ', '_'); + + container.innerHTML = ''; + + // Header + const header = document.createElement('div'); + header.className = 'job-detail-header'; + + let headerHtml = '' + + '

' + escapeHtml(job.title) + '

' + + '' + escapeHtml(job.state) + ''; + + if (job.state === 'failed' || job.state === 'interrupted') { + headerHtml += ''; + } + if (job.browse_url) { + headerHtml += 'Browse Files'; + } + + header.innerHTML = headerHtml; + container.appendChild(header); + + // Sub-tab bar + const tabs = document.createElement('div'); + tabs.className = 'job-detail-tabs'; + const subtabs = ['overview', 'activity', 'files']; + for (const st of subtabs) { + const btn = document.createElement('button'); + btn.textContent = st.charAt(0).toUpperCase() + st.slice(1); + btn.className = st === currentJobSubTab ? 'active' : ''; + btn.addEventListener('click', () => { + currentJobSubTab = st; + renderJobDetail(job); + }); + tabs.appendChild(btn); + } + container.appendChild(tabs); + + // Content + const content = document.createElement('div'); + content.className = 'job-detail-content'; + container.appendChild(content); + + switch (currentJobSubTab) { + case 'overview': renderJobOverview(content, job); break; + case 'files': renderJobFiles(content, job); break; + case 'activity': renderJobActivity(content, job); break; + } +} + +function metaItem(label, value) { + return '
' + escapeHtml(label) + + '
' + escapeHtml(String(value != null ? value : '-')) + + '
'; +} + +function formatDuration(secs) { + if (secs == null) return '-'; + if (secs < 60) return secs + 's'; + const m = Math.floor(secs / 60); + const s = secs % 60; + if (m < 60) return m + 'm ' + s + 's'; + const h = Math.floor(m / 60); + return h + 'h ' + (m % 60) + 'm'; +} + +function renderJobOverview(container, job) { + // Metadata grid + const grid = document.createElement('div'); + grid.className = 'job-meta-grid'; + grid.innerHTML = metaItem('Job ID', job.id) + + metaItem('State', job.state) + + metaItem('Created', formatDate(job.created_at)) + + metaItem('Started', formatDate(job.started_at)) + + metaItem('Completed', formatDate(job.completed_at)) + + metaItem('Duration', formatDuration(job.elapsed_secs)) + + (job.job_mode ? metaItem('Mode', job.job_mode) : ''); + container.appendChild(grid); + + // Description + if (job.description) { + const descSection = document.createElement('div'); + descSection.className = 'job-description'; + const descHeader = document.createElement('h3'); + descHeader.textContent = 'Description'; + descSection.appendChild(descHeader); + const descBody = document.createElement('div'); + descBody.className = 'job-description-body'; + descBody.innerHTML = renderMarkdown(job.description); + descSection.appendChild(descBody); + container.appendChild(descSection); + } + + // State transitions timeline + if (job.transitions.length > 0) { + const timelineSection = document.createElement('div'); + timelineSection.className = 'job-timeline-section'; + const tlHeader = document.createElement('h3'); + tlHeader.textContent = 'State Transitions'; + timelineSection.appendChild(tlHeader); + + const timeline = document.createElement('div'); + timeline.className = 'timeline'; + for (const t of job.transitions) { + const entry = document.createElement('div'); + entry.className = 'timeline-entry'; + const dot = document.createElement('div'); + dot.className = 'timeline-dot'; + entry.appendChild(dot); + const info = document.createElement('div'); + info.className = 'timeline-info'; + info.innerHTML = '' + escapeHtml(t.from) + '' + + ' → ' + + '' + escapeHtml(t.to) + '' + + '' + formatDate(t.timestamp) + '' + + (t.reason ? '
' + escapeHtml(t.reason) + '
' : ''); + entry.appendChild(info); + timeline.appendChild(entry); + } + timelineSection.appendChild(timeline); + container.appendChild(timelineSection); + } +} + +function renderJobFiles(container, job) { + container.innerHTML = '
' + + '
' + + '
Select a file to view
' + + '
'; + + container._jobId = job ? job.id : null; + + apiFetch('/api/jobs/' + job.id + '/files/list?path=').then((data) => { + jobFilesTreeState = data.entries.map((e) => ({ + name: e.name, + path: e.path, + is_dir: e.is_dir, + children: e.is_dir ? null : undefined, + expanded: false, + loaded: false, + })); + renderJobFilesTree(); + }).catch(() => { + const treeContainer = document.querySelector('.job-files-tree'); + if (treeContainer) { + treeContainer.innerHTML = '
No project files
'; + } + }); +} + +function renderJobFilesTree() { + const treeContainer = document.querySelector('.job-files-tree'); + if (!treeContainer) return; + treeContainer.innerHTML = ''; + if (!jobFilesTreeState || jobFilesTreeState.length === 0) { + treeContainer.innerHTML = '
No files in workspace
'; + return; + } + renderJobFileNodes(jobFilesTreeState, treeContainer, 0); +} + +function renderJobFileNodes(nodes, container, depth) { + for (const node of nodes) { + const row = document.createElement('div'); + row.className = 'tree-row'; + row.style.paddingLeft = (depth * 16 + 8) + 'px'; + + if (node.is_dir) { + const arrow = document.createElement('span'); + arrow.className = 'expand-arrow' + (node.expanded ? ' expanded' : ''); + arrow.textContent = '\u25B6'; + arrow.addEventListener('click', (e) => { + e.stopPropagation(); + toggleJobFileExpand(node); + }); + row.appendChild(arrow); + + const label = document.createElement('span'); + label.className = 'tree-label dir'; + label.textContent = node.name; + label.addEventListener('click', () => toggleJobFileExpand(node)); + row.appendChild(label); + } else { + const spacer = document.createElement('span'); + spacer.className = 'expand-arrow-spacer'; + row.appendChild(spacer); + + const label = document.createElement('span'); + label.className = 'tree-label file'; + label.textContent = node.name; + label.addEventListener('click', () => readJobFile(node.path)); + row.appendChild(label); + } + + container.appendChild(row); + + if (node.is_dir && node.expanded && node.children) { + const childContainer = document.createElement('div'); + childContainer.className = 'tree-children'; + renderJobFileNodes(node.children, childContainer, depth + 1); + container.appendChild(childContainer); + } + } +} + +function getJobId() { + const container = document.querySelector('.job-detail-content'); + return (container && container._jobId) || null; +} + +function toggleJobFileExpand(node) { + if (node.expanded) { + node.expanded = false; + renderJobFilesTree(); + return; + } + if (node.loaded) { + node.expanded = true; + renderJobFilesTree(); + return; + } + const jobId = getJobId(); + apiFetch('/api/jobs/' + jobId + '/files/list?path=' + encodeURIComponent(node.path)).then((data) => { + node.children = data.entries.map((e) => ({ + name: e.name, + path: e.path, + is_dir: e.is_dir, + children: e.is_dir ? null : undefined, + expanded: false, + loaded: false, + })); + node.loaded = true; + node.expanded = true; + renderJobFilesTree(); + }).catch(() => {}); +} + +function readJobFile(path) { + const viewer = document.querySelector('.job-files-viewer'); + if (!viewer) return; + const jobId = getJobId(); + apiFetch('/api/jobs/' + jobId + '/files/read?path=' + encodeURIComponent(path)).then((data) => { + viewer.innerHTML = '
' + escapeHtml(path) + '
' + + '
' + escapeHtml(data.content) + '
'; + }).catch((err) => { + viewer.innerHTML = '
Error: ' + escapeHtml(err.message) + '
'; + }); +} + +// --- Activity tab (unified for all sandbox jobs) --- + +let activityCurrentJobId = null; +// Track how many live SSE events we've already rendered so refreshActivityTab +// only appends new ones (avoids duplicates on each SSE tick). +let activityRenderedLiveIndex = 0; + +function renderJobActivity(container, job) { + activityCurrentJobId = job ? job.id : null; + activityRenderedLiveIndex = 0; + + container.innerHTML = '
' + + '' + + '' + + '
' + + '
' + + '
' + + '' + + '' + + '' + + '
'; + + document.getElementById('activity-type-filter').addEventListener('change', applyActivityFilter); + + const terminal = document.getElementById('activity-terminal'); + const input = document.getElementById('activity-prompt-input'); + const sendBtn = document.getElementById('activity-send-btn'); + const doneBtn = document.getElementById('activity-done-btn'); + + sendBtn.addEventListener('click', () => sendJobPrompt(job.id, false)); + doneBtn.addEventListener('click', () => sendJobPrompt(job.id, true)); + input.addEventListener('keydown', (e) => { + if (e.key === 'Enter') sendJobPrompt(job.id, false); + }); + + // Load persisted events from DB, then catch up with any live SSE events + apiFetch('/api/jobs/' + job.id + '/events').then((data) => { + if (data.events && data.events.length > 0) { + for (const evt of data.events) { + appendActivityEvent(terminal, evt.event_type, evt.data); + } + } + appendNewLiveEvents(terminal, job.id); + }).catch(() => { + appendNewLiveEvents(terminal, job.id); + }); +} + +function appendNewLiveEvents(terminal, jobId) { + const live = jobEvents.get(jobId) || []; + for (let i = activityRenderedLiveIndex; i < live.length; i++) { + const evt = live[i]; + appendActivityEvent(terminal, evt.type.replace('job_', ''), evt.data); + } + activityRenderedLiveIndex = live.length; + const autoScroll = document.getElementById('activity-autoscroll'); + if (!autoScroll || autoScroll.checked) { + terminal.scrollTop = terminal.scrollHeight; + } +} + +function applyActivityFilter() { + const filter = document.getElementById('activity-type-filter').value; + const events = document.querySelectorAll('#activity-terminal .activity-event'); + for (const el of events) { + if (filter === 'all') { + el.style.display = ''; + } else { + el.style.display = el.getAttribute('data-event-type') === filter ? '' : 'none'; + } + } +} + +function appendActivityEvent(terminal, eventType, data) { + if (!terminal) return; + const el = document.createElement('div'); + el.className = 'activity-event activity-event-' + eventType; + el.setAttribute('data-event-type', eventType); + + // Respect current filter + const filterEl = document.getElementById('activity-type-filter'); + if (filterEl && filterEl.value !== 'all' && filterEl.value !== eventType) { + el.style.display = 'none'; + } + + switch (eventType) { + case 'message': + el.innerHTML = '' + escapeHtml(data.role || 'assistant') + ' ' + + '' + escapeHtml(data.content || '') + ''; + break; + case 'tool_use': + el.innerHTML = '
' + + ' ' + + escapeHtml(data.tool_name || 'tool') + + '
'
+        + escapeHtml(typeof data.input === 'string' ? data.input : JSON.stringify(data.input, null, 2))
+        + '
'; + break; + case 'tool_result': + el.innerHTML = '
' + + ' ' + + escapeHtml(data.tool_name || 'result') + + '
'
+        + escapeHtml(data.output || '')
+        + '
'; + break; + case 'status': + el.innerHTML = '' + escapeHtml(data.message || '') + ''; + break; + case 'result': + el.className += ' activity-final'; + const success = data.success !== false; + el.innerHTML = '' + + escapeHtml(data.message || data.status || 'done') + ''; + if (data.session_id) { + el.innerHTML += ' session: ' + escapeHtml(data.session_id) + ''; + } + break; + default: + el.innerHTML = '' + escapeHtml(JSON.stringify(data)) + ''; + } + + terminal.appendChild(el); +} + +function refreshActivityTab(jobId) { + if (activityCurrentJobId !== jobId) return; + if (currentJobSubTab !== 'activity') return; + const terminal = document.getElementById('activity-terminal'); + if (!terminal) return; + appendNewLiveEvents(terminal, jobId); +} + +function sendJobPrompt(jobId, done) { + const input = document.getElementById('activity-prompt-input'); + const content = input ? input.value.trim() : ''; + if (!content && !done) return; + + apiFetch('/api/jobs/' + jobId + '/prompt', { + method: 'POST', + body: { content: content || '(done)', done: done }, + }).then(() => { + if (input) input.value = ''; + if (done) { + const bar = document.getElementById('activity-input-bar'); + if (bar) bar.innerHTML = 'Done signal sent'; + } + }).catch((err) => { + const terminal = document.getElementById('activity-terminal'); + if (terminal) { + appendActivityEvent(terminal, 'status', { message: 'Failed to send: ' + err.message }); + } + }); +} + +// --- Routines --- + +let currentRoutineId = null; + +function loadRoutines() { + currentRoutineId = null; + + // Restore list view if detail was open + const detail = document.getElementById('routine-detail'); + if (detail) detail.style.display = 'none'; + const table = document.getElementById('routines-table'); + if (table) table.style.display = ''; + + Promise.all([ + apiFetch('/api/routines/summary'), + apiFetch('/api/routines'), + ]).then(([summary, listData]) => { + renderRoutinesSummary(summary); + renderRoutinesList(listData.routines); + }).catch(() => {}); +} + +function renderRoutinesSummary(s) { + document.getElementById('routines-summary').innerHTML = '' + + summaryCard('Total', s.total, '') + + summaryCard('Enabled', s.enabled, 'active') + + summaryCard('Disabled', s.disabled, '') + + summaryCard('Failing', s.failing, 'failed') + + summaryCard('Runs Today', s.runs_today, 'completed'); +} + +function renderRoutinesList(routines) { + const tbody = document.getElementById('routines-tbody'); + const empty = document.getElementById('routines-empty'); + + if (!routines || routines.length === 0) { + tbody.innerHTML = ''; + empty.style.display = 'block'; + return; + } + + empty.style.display = 'none'; + tbody.innerHTML = routines.map((r) => { + const statusClass = r.status === 'active' ? 'completed' + : r.status === 'failing' ? 'failed' + : 'pending'; + + const toggleLabel = r.enabled ? 'Disable' : 'Enable'; + const toggleClass = r.enabled ? 'btn-cancel' : 'btn-restart'; + + return '' + + '' + escapeHtml(r.name) + '' + + '' + escapeHtml(r.trigger_summary) + '' + + '' + escapeHtml(r.action_type) + '' + + '' + formatRelativeTime(r.last_run_at) + '' + + '' + formatRelativeTime(r.next_fire_at) + '' + + '' + r.run_count + '' + + '' + escapeHtml(r.status) + '' + + '' + + ' ' + + ' ' + + '' + + '' + + ''; + }).join(''); +} + +function openRoutineDetail(id) { + currentRoutineId = id; + apiFetch('/api/routines/' + id).then((routine) => { + renderRoutineDetail(routine); + }).catch((err) => { + showToast('Failed to load routine: ' + err.message, 'error'); + }); +} + +function closeRoutineDetail() { + currentRoutineId = null; + loadRoutines(); +} + +function renderRoutineDetail(routine) { + const table = document.getElementById('routines-table'); + if (table) table.style.display = 'none'; + document.getElementById('routines-empty').style.display = 'none'; + + const detail = document.getElementById('routine-detail'); + detail.style.display = 'block'; + + const statusClass = !routine.enabled ? 'pending' + : routine.consecutive_failures > 0 ? 'failed' + : 'completed'; + const statusLabel = !routine.enabled ? 'disabled' + : routine.consecutive_failures > 0 ? 'failing' + : 'active'; + + let html = '
' + + '' + + '

' + escapeHtml(routine.name) + '

' + + '' + escapeHtml(statusLabel) + '' + + '
'; + + // Metadata grid + html += '
' + + metaItem('Routine ID', routine.id) + + metaItem('Enabled', routine.enabled ? 'Yes' : 'No') + + metaItem('Run Count', routine.run_count) + + metaItem('Failures', routine.consecutive_failures) + + metaItem('Last Run', formatDate(routine.last_run_at)) + + metaItem('Next Fire', formatDate(routine.next_fire_at)) + + metaItem('Created', formatDate(routine.created_at)) + + '
'; + + // Description + if (routine.description) { + html += '

Description

' + + '
' + escapeHtml(routine.description) + '
'; + } + + // Trigger config + html += '

Trigger

' + + '
' + escapeHtml(JSON.stringify(routine.trigger, null, 2)) + '
'; + + // Action config + html += '

Action

' + + '
' + escapeHtml(JSON.stringify(routine.action, null, 2)) + '
'; + + // Recent runs + if (routine.recent_runs && routine.recent_runs.length > 0) { + html += '

Recent Runs

' + + '' + + '' + + ''; + for (const run of routine.recent_runs) { + const runStatusClass = run.status === 'Ok' ? 'completed' + : run.status === 'Failed' ? 'failed' + : run.status === 'Attention' ? 'stuck' + : 'in_progress'; + html += '' + + '' + + '' + + '' + + '' + + '' + + '' + + ''; + } + html += '
TriggerStartedCompletedStatusSummaryTokens
' + escapeHtml(run.trigger_type) + '' + formatDate(run.started_at) + '' + formatDate(run.completed_at) + '' + escapeHtml(run.status) + '' + escapeHtml(run.result_summary || '-') + '' + (run.tokens_used != null ? run.tokens_used : '-') + '
'; + } + + detail.innerHTML = html; +} + +function triggerRoutine(id) { + apiFetch('/api/routines/' + id + '/trigger', { method: 'POST' }) + .then(() => showToast('Routine triggered', 'success')) + .catch((err) => showToast('Trigger failed: ' + err.message, 'error')); +} + +function toggleRoutine(id) { + apiFetch('/api/routines/' + id + '/toggle', { method: 'POST' }) + .then((res) => { + showToast('Routine ' + (res.status || 'toggled'), 'success'); + if (currentRoutineId) openRoutineDetail(currentRoutineId); + else loadRoutines(); + }) + .catch((err) => showToast('Toggle failed: ' + err.message, 'error')); +} + +function deleteRoutine(id, name) { + if (!confirm('Delete routine "' + name + '"?')) return; + apiFetch('/api/routines/' + id, { method: 'DELETE' }) + .then(() => { + showToast('Routine deleted', 'success'); + if (currentRoutineId === id) closeRoutineDetail(); + else loadRoutines(); + }) + .catch((err) => showToast('Delete failed: ' + err.message, 'error')); +} + +function formatRelativeTime(isoString) { + if (!isoString) return '-'; + const d = new Date(isoString); + const now = Date.now(); + const diffMs = now - d.getTime(); + const absDiff = Math.abs(diffMs); + const future = diffMs < 0; + + if (absDiff < 60000) return future ? 'in <1m' : '<1m ago'; + if (absDiff < 3600000) { + const m = Math.floor(absDiff / 60000); + return future ? 'in ' + m + 'm' : m + 'm ago'; + } + if (absDiff < 86400000) { + const h = Math.floor(absDiff / 3600000); + return future ? 'in ' + h + 'h' : h + 'h ago'; + } + const days = Math.floor(absDiff / 86400000); + return future ? 'in ' + days + 'd' : days + 'd ago'; +} + +// --- Gateway status widget --- + +let gatewayStatusInterval = null; + +function startGatewayStatusPolling() { + fetchGatewayStatus(); + gatewayStatusInterval = setInterval(fetchGatewayStatus, 30000); +} + +function fetchGatewayStatus() { + apiFetch('/api/gateway/status').then((data) => { + const popover = document.getElementById('gateway-popover'); + popover.innerHTML = '
SSE clients' + (data.sse_clients || 0) + '
' + + '
Log clients' + (data.log_clients || 0) + '
' + + '
Uptime' + formatDuration(data.uptime_secs) + '
'; + }).catch(() => {}); +} + +// Show/hide popover on hover +document.getElementById('gateway-status-trigger').addEventListener('mouseenter', () => { + document.getElementById('gateway-popover').classList.add('visible'); +}); +document.getElementById('gateway-status-trigger').addEventListener('mouseleave', () => { + document.getElementById('gateway-popover').classList.remove('visible'); +}); + +// --- Extension install --- + +function installExtension() { + const name = document.getElementById('ext-install-name').value.trim(); + if (!name) { + showToast('Extension name is required', 'error'); + return; + } + const url = document.getElementById('ext-install-url').value.trim(); + const kind = document.getElementById('ext-install-kind').value; + + apiFetch('/api/extensions/install', { + method: 'POST', + body: { name, url: url || undefined, kind }, + }).then((res) => { + if (res.success) { + showToast('Installed ' + name, 'success'); + document.getElementById('ext-install-name').value = ''; + document.getElementById('ext-install-url').value = ''; + loadExtensions(); + } else { + showToast('Install failed: ' + (res.message || 'unknown error'), 'error'); + } + }).catch((err) => { + showToast('Install failed: ' + err.message, 'error'); + }); +} + +// --- Keyboard shortcuts --- + +document.addEventListener('keydown', (e) => { + const mod = e.metaKey || e.ctrlKey; + const tag = (e.target.tagName || '').toLowerCase(); + const inInput = tag === 'input' || tag === 'textarea'; + + // Mod+1-6: switch tabs + if (mod && e.key >= '1' && e.key <= '6') { + e.preventDefault(); + const tabs = ['chat', 'memory', 'jobs', 'routines', 'logs', 'extensions']; + const idx = parseInt(e.key) - 1; + if (tabs[idx]) switchTab(tabs[idx]); + return; + } + + // Mod+K: focus chat input or memory search + if (mod && e.key === 'k') { + e.preventDefault(); + if (currentTab === 'memory') { + document.getElementById('memory-search').focus(); + } else { + document.getElementById('chat-input').focus(); + } + return; + } + + // Mod+N: new thread + if (mod && e.key === 'n' && currentTab === 'chat') { + e.preventDefault(); + createNewThread(); + return; + } + + // Escape: close job detail or blur input + if (e.key === 'Escape') { + if (currentJobId) { + closeJobDetail(); + } else if (inInput) { + e.target.blur(); + } + return; + } +}); + +// --- Toasts --- + +function showToast(message, type) { + const container = document.getElementById('toasts'); + const toast = document.createElement('div'); + toast.className = 'toast toast-' + (type || 'info'); + toast.textContent = message; + container.appendChild(toast); + // Trigger slide-in + requestAnimationFrame(() => toast.classList.add('visible')); + setTimeout(() => { + toast.classList.remove('visible'); + toast.addEventListener('transitionend', () => toast.remove()); + }, 4000); +} + // --- Utilities --- function escapeHtml(str) { diff --git a/src/channels/web/static/index.html b/src/channels/web/static/index.html index 8ae5be51..bf6c227c 100644 --- a/src/channels/web/static/index.html +++ b/src/channels/web/static/index.html @@ -10,12 +10,19 @@
-

IronClaw

-
- - + -
@@ -26,16 +33,33 @@ +
-
+
Connected +
+
+
+ Threads + + +
+
+ Assistant + +
+
+ Conversations +
+
+
@@ -56,10 +80,20 @@
-
workspace /
+
+ workspace / + +
Select a file to view its contents
+
@@ -73,6 +107,7 @@ ID Title + Source Status Created Actions @@ -104,9 +139,48 @@ + +
+
+
+ + + + + + + + + + + + + + +
NameTriggerActionLast RunNext RunRunsStatusActions
+ + +
+
+
+
+

Install Extension

+
+ + + + +
+

Installed Extensions

@@ -130,6 +204,7 @@
+
diff --git a/src/channels/web/static/style.css b/src/channels/web/static/style.css index bb30ccb2..d28edc2b 100644 --- a/src/channels/web/static/style.css +++ b/src/channels/web/static/style.css @@ -39,28 +39,57 @@ body { align-items: center; justify-content: center; height: 100vh; - flex-direction: column; - gap: 16px; } -#auth-screen h1 { - font-size: 24px; - font-weight: 600; +.auth-card-login { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: 12px; + padding: 40px 36px 32px; + width: 100%; + max-width: 400px; + display: flex; + flex-direction: column; + gap: 24px; + box-shadow: 0 4px 24px rgba(0, 0, 0, 0.3); +} + +.auth-brand { + text-align: center; +} + +.auth-brand h1 { + font-size: 28px; + font-weight: 700; + color: var(--text); + margin-bottom: 4px; +} + +.auth-tagline { + font-size: 14px; + color: var(--text-secondary); } #auth-screen .auth-form { display: flex; + flex-direction: column; gap: 8px; } +#auth-screen .auth-form label { + font-size: 13px; + font-weight: 500; + color: var(--text-secondary); +} + #auth-screen input { - padding: 8px 12px; - background: var(--bg-secondary); + padding: 10px 12px; + background: var(--bg); border: 1px solid var(--border); border-radius: var(--radius); color: var(--text); font-size: 14px; - width: 300px; + width: 100%; } #auth-screen input:focus { @@ -69,13 +98,15 @@ body { } #auth-screen button { - padding: 8px 16px; + padding: 10px 16px; background: var(--accent); color: #fff; border: none; border-radius: var(--radius); cursor: pointer; font-size: 14px; + font-weight: 500; + margin-top: 4px; } #auth-screen button:hover { @@ -86,6 +117,14 @@ body { color: var(--danger); font-size: 13px; min-height: 20px; + text-align: center; +} + +.auth-hint { + font-size: 12px; + color: var(--text-secondary); + text-align: center; + line-height: 1.4; } /* Main App */ @@ -135,6 +174,8 @@ body { gap: 8px; font-size: 12px; color: var(--text-secondary); + position: relative; + cursor: pointer; } .tab-bar .status .dot { @@ -283,6 +324,25 @@ body { to { transform: rotate(360deg); } } +.scroll-load-spinner { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + padding: 8px; + color: var(--text-secondary); + font-size: 12px; +} + +.scroll-load-spinner .spinner { + width: 12px; + height: 12px; + border: 2px solid var(--border); + border-top-color: var(--accent); + border-radius: 50%; + animation: spin 0.6s linear infinite; +} + /* Approval card (inline in chat) */ .approval-card { align-self: flex-start; @@ -391,6 +451,109 @@ body { font-style: italic; } +/* Auth card (inline in chat) */ +.auth-card { + align-self: flex-start; + max-width: 80%; + background: var(--bg-secondary); + border: 1px solid var(--accent); + border-radius: var(--radius); + padding: 12px 16px; + margin: 8px 0; + display: flex; + flex-direction: column; + gap: 8px; +} + +.auth-card .auth-header { + font-weight: 600; + color: var(--accent); + font-size: 13px; +} + +.auth-card .auth-instructions { + font-size: 13px; + color: var(--text); + line-height: 1.4; +} + +.auth-card .auth-links { + display: flex; + gap: 8px; + align-items: center; +} + +.auth-card .auth-links a { + color: var(--accent); + font-size: 13px; + text-decoration: underline; +} + +.auth-card .auth-token-input { + display: flex; + gap: 8px; + align-items: center; +} + +.auth-card .auth-token-input input { + flex: 1; + padding: 6px 10px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--bg); + color: var(--text); + font-size: 13px; + font-family: monospace; +} + +.auth-card .auth-token-input input:focus { + outline: none; + border-color: var(--accent); +} + +.auth-card .auth-actions { + display: flex; + gap: 8px; + align-items: center; +} + +.auth-card .auth-actions button { + padding: 6px 14px; + border: 1px solid var(--border); + border-radius: var(--radius); + cursor: pointer; + font-size: 13px; + background: var(--bg-secondary); + color: var(--text); +} + +.auth-card .auth-actions button:disabled { + opacity: 0.4; + cursor: not-allowed; +} + +.auth-card .auth-actions button.auth-submit { + background: var(--accent); + border-color: var(--accent); + color: #fff; +} + +.auth-card .auth-actions button.auth-cancel { + background: var(--bg-secondary); + border-color: var(--border); +} + +.auth-card .auth-actions button.auth-oauth { + background: var(--success); + border-color: var(--success); + color: #fff; +} + +.auth-card .auth-error { + color: var(--danger); + font-size: 12px; +} + /* Chat input */ .chat-input { display: flex; @@ -583,6 +746,9 @@ body { color: var(--text-secondary); border-bottom: 1px solid var(--border); background: var(--bg-secondary); + display: flex; + align-items: center; + gap: 8px; } .memory-breadcrumb a { @@ -718,6 +884,9 @@ body { .badge.failed { background: rgba(248, 81, 73, 0.15); color: var(--danger); } .badge.stuck { background: rgba(210, 153, 34, 0.15); color: var(--warning); } .badge.cancelled { background: var(--bg-tertiary); color: var(--text-secondary); } +.badge.interrupted { background: rgba(210, 153, 34, 0.15); color: var(--warning); } +.badge.source-sandbox { background: rgba(136, 132, 216, 0.15); color: #b4b0e8; } +.badge.source-direct { background: var(--bg-tertiary); color: var(--text-secondary); } .btn-cancel { padding: 4px 10px; @@ -733,12 +902,587 @@ body { background: rgba(248, 81, 73, 0.15); } +.btn-restart { + padding: 4px 10px; + background: none; + border: 1px solid var(--accent); + border-radius: var(--radius); + color: var(--accent); + cursor: pointer; + font-size: 12px; +} + +.btn-restart:hover { + background: rgba(88, 166, 255, 0.15); +} + +.btn-browse { + padding: 4px 10px; + background: none; + border: 1px solid var(--success); + border-radius: var(--radius); + color: var(--success); + cursor: pointer; + font-size: 12px; + text-decoration: none; +} + +.btn-browse:hover { + background: rgba(63, 185, 80, 0.15); +} + +/* Job started card in chat */ +.job-card { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + margin: 8px 0; + background: var(--bg-tertiary); + border: 1px solid var(--border); + border-radius: var(--radius); + border-left: 3px solid var(--accent); +} + +.job-card-icon { + font-size: 20px; +} + +.job-card-info { + flex: 1; +} + +.job-card-title { + font-weight: 600; + font-size: 14px; +} + +.job-card-id { + font-size: 12px; + color: var(--text-secondary); + font-family: monospace; +} + +.job-card-view, .job-card-browse { + padding: 4px 12px; + border-radius: var(--radius); + font-size: 12px; + cursor: pointer; + text-decoration: none; +} + +.job-card-view { + background: none; + border: 1px solid var(--accent); + color: var(--accent); +} + +.job-card-view:hover { + background: rgba(88, 166, 255, 0.15); +} + +.job-card-browse { + background: none; + border: 1px solid var(--success); + color: var(--success); +} + +.job-card-browse:hover { + background: rgba(63, 185, 80, 0.15); +} + +/* Clickable job rows */ +.job-row { + cursor: pointer; +} + +/* Job Detail View */ +.job-detail-header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 16px; +} + +.job-detail-header h2 { + font-size: 18px; + font-weight: 600; + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.btn-back { + padding: 6px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + cursor: pointer; + font-size: 13px; + flex-shrink: 0; +} + +.btn-back:hover { + background: var(--bg-tertiary); +} + +.job-detail-tabs { + display: flex; + gap: 0; + border-bottom: 1px solid var(--border); + margin-bottom: 16px; +} + +.job-detail-tabs button { + padding: 8px 16px; + background: none; + border: none; + border-bottom: 2px solid transparent; + color: var(--text-secondary); + cursor: pointer; + font-size: 13px; +} + +.job-detail-tabs button:hover { + color: var(--text); +} + +.job-detail-tabs button.active { + color: var(--accent); + border-bottom-color: var(--accent); +} + +.job-detail-content { + flex: 1; + overflow-y: auto; +} + +/* Metadata grid */ +.job-meta-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); + gap: 12px; + margin-bottom: 20px; +} + +.meta-item { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 10px 12px; +} + +.meta-label { + font-size: 11px; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 4px; +} + +.meta-value { + font-size: 14px; + color: var(--text); + word-break: break-all; +} + +/* Job description */ +.job-description { + margin-bottom: 20px; +} + +.job-description h3 { + font-size: 14px; + font-weight: 600; + margin-bottom: 8px; + color: var(--text); +} + +.job-description-body { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 12px 16px; + font-size: 14px; + line-height: 1.6; +} + +/* State transitions timeline */ +.job-timeline-section { + margin-bottom: 20px; +} + +.job-timeline-section h3 { + font-size: 14px; + font-weight: 600; + margin-bottom: 12px; + color: var(--text); +} + +.timeline { + position: relative; + padding-left: 20px; + border-left: 2px solid var(--border); +} + +.timeline-entry { + position: relative; + padding: 8px 0 8px 16px; +} + +.timeline-dot { + position: absolute; + left: -27px; + top: 14px; + width: 10px; + height: 10px; + border-radius: 50%; + background: var(--accent); + border: 2px solid var(--bg); +} + +.timeline-info { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 6px; + font-size: 13px; +} + +.timeline-time { + color: var(--text-secondary); + font-size: 12px; + margin-left: 8px; +} + +.timeline-reason { + width: 100%; + font-size: 12px; + color: var(--text-secondary); + margin-top: 2px; +} + +/* Action cards */ +.action-card { + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + margin-bottom: 8px; + border-left: 3px solid var(--success); +} + +.action-card.failure { + border-left-color: var(--danger); +} + +.action-header { + display: flex; + align-items: center; + gap: 10px; + padding: 10px 12px; + cursor: pointer; + font-size: 13px; +} + +.action-header:hover { + background: var(--bg-tertiary); +} + +.action-tool { + font-weight: 600; + color: var(--text); + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; +} + +.action-seq { + color: var(--text-secondary); + font-size: 11px; +} + +.action-duration { + color: var(--text-secondary); + font-size: 12px; +} + +.action-time { + color: var(--text-secondary); + font-size: 12px; + margin-left: auto; +} + +.action-toggle { + color: var(--text-secondary); + font-size: 10px; + flex-shrink: 0; +} + +.action-detail { + padding: 0 12px 12px; +} + +.action-section { + margin-top: 8px; +} + +.action-section strong { + font-size: 12px; + color: var(--text-secondary); + display: block; + margin-bottom: 4px; +} + +.action-json { + background: var(--code-bg); + padding: 8px 12px; + border-radius: var(--radius); + font-size: 12px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + line-height: 1.4; + overflow-x: auto; + color: var(--text-secondary); + margin: 0; + white-space: pre-wrap; + word-break: break-all; + max-height: 300px; + overflow-y: auto; +} + +.action-error { + background: rgba(248, 81, 73, 0.1); + padding: 8px 12px; + border-radius: var(--radius); + font-size: 12px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + line-height: 1.4; + color: var(--danger); + margin: 0; + white-space: pre-wrap; + word-break: break-all; +} + +/* Conversation messages */ +.conv-message { + padding: 10px 14px; + border-radius: var(--radius); + margin-bottom: 8px; + font-size: 14px; + line-height: 1.5; +} + +.conv-role { + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 4px; +} + +.conv-body { + word-wrap: break-word; +} + +.conv-system { + background: var(--bg-tertiary); + border: 1px solid var(--border); +} + +.conv-system .conv-role { color: var(--text-secondary); } +.conv-system .conv-body { color: var(--text-secondary); font-size: 13px; } + +.conv-user { + background: rgba(88, 166, 255, 0.08); + border: 1px solid rgba(88, 166, 255, 0.2); +} + +.conv-user .conv-role { color: var(--accent); } + +.conv-assistant { + background: var(--bg-secondary); + border: 1px solid var(--border); +} + +.conv-assistant .conv-role { color: var(--success); } + +.conv-tool { + background: var(--bg-secondary); + border: 1px solid var(--border); + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + font-size: 13px; +} + +.conv-tool .conv-role { color: var(--warning); } +.conv-tool .conv-body { white-space: pre-wrap; word-break: break-all; max-height: 200px; overflow-y: auto; } + +.conv-tc-id { + font-size: 11px; + color: var(--text-secondary); + margin-bottom: 4px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; +} + +.conv-tool-calls { + margin-top: 8px; + border-top: 1px solid var(--border); + padding-top: 8px; +} + +.conv-tc-entry { + margin-bottom: 6px; +} + +.conv-tc-name { + font-size: 12px; + font-weight: 600; + color: var(--accent); + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; +} + +.conv-tc-args { + background: var(--code-bg); + padding: 6px 10px; + border-radius: var(--radius); + font-size: 11px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + line-height: 1.4; + margin: 4px 0 0; + color: var(--text-secondary); + white-space: pre-wrap; + word-break: break-all; + max-height: 150px; + overflow-y: auto; +} + +/* Job files browser */ +.job-files { + display: flex; + height: calc(100vh - 280px); + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; +} + +.job-files-sidebar { + width: 240px; + border-right: 1px solid var(--border); + background: var(--bg-secondary); + overflow-y: auto; +} + +.job-files-tree { + padding: 8px 0; +} + +.job-files-viewer { + flex: 1; + overflow: auto; + padding: 12px 16px; +} + +.job-files-path { + font-size: 12px; + color: var(--accent); + margin-bottom: 8px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; +} + +.job-files-content { + font-size: 13px; + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-all; + color: var(--text); + margin: 0; +} + .empty-state { text-align: center; padding: 40px; color: var(--text-secondary); } +/* Routines Tab */ +.routines-container { + flex: 1; + overflow-y: auto; + padding: 16px; +} + +.routines-summary { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 12px; + margin-bottom: 20px; +} + +.routines-table { + width: 100%; + border-collapse: collapse; +} + +.routines-table th, +.routines-table td { + padding: 10px 12px; + text-align: left; + border-bottom: 1px solid var(--border); + font-size: 13px; +} + +.routines-table th { + color: var(--text-secondary); + font-weight: 500; + text-transform: uppercase; + font-size: 11px; + letter-spacing: 0.5px; +} + +.routines-table tr:hover td { + background: var(--bg-secondary); +} + +.routine-row { + cursor: pointer; +} + +.routine-detail { + padding: 16px 0; +} + +.badge.enabled { background: rgba(63, 185, 80, 0.15); color: var(--success); } +.badge.disabled { background: var(--bg-tertiary); color: var(--text-secondary); } +.badge.failing { background: rgba(248, 81, 73, 0.15); color: var(--danger); } + +.btn-trigger { + padding: 4px 10px; + background: none; + border: 1px solid var(--accent); + border-radius: var(--radius); + color: var(--accent); + cursor: pointer; + font-size: 12px; +} + +.btn-trigger:hover { + background: rgba(88, 166, 255, 0.15); +} + +.btn-toggle { + padding: 4px 10px; + background: none; + border: 1px solid var(--warning); + border-radius: var(--radius); + color: var(--warning); + cursor: pointer; + font-size: 12px; +} + +.btn-toggle:hover { + background: rgba(210, 153, 34, 0.15); +} + /* Logs Tab */ .logs-container { flex: 1; @@ -1050,3 +1794,710 @@ body { .tools-table tr:hover td { background: var(--bg-secondary); } + +/* --- Activity tab (unified sandbox job events) --- */ + +.activity-terminal { + flex: 1; + overflow-y: auto; + padding: 12px; + font-family: monospace; + font-size: 13px; + line-height: 1.6; + background: var(--bg); + border: 1px solid var(--border); + border-radius: 6px; + margin-bottom: 8px; + max-height: calc(100vh - 320px); +} + +.activity-event { + padding: 4px 0; + border-bottom: 1px solid rgba(255, 255, 255, 0.04); +} + +.activity-event-message .activity-role { + color: var(--accent); + font-weight: 600; + margin-right: 8px; +} + +.activity-event-message .activity-content { + white-space: pre-wrap; + word-break: break-word; +} + +.activity-event-status .activity-status { + color: var(--text-secondary); + font-style: italic; +} + +.activity-event-result.activity-final { + padding: 8px 0; + font-weight: 600; +} + +.activity-result-status { + color: var(--success); +} + +.activity-result-status[data-success="false"] { + color: var(--danger); +} + +.activity-session-id { + color: var(--text-secondary); + font-size: 11px; + font-weight: 400; +} + +.activity-tool-block { + margin: 4px 0; + border: 1px solid var(--border); + border-radius: 4px; + overflow: hidden; +} + +.activity-tool-block summary { + padding: 6px 10px; + cursor: pointer; + background: var(--bg-secondary); + font-size: 12px; + color: var(--text-secondary); +} + +.activity-tool-block summary:hover { + color: var(--text); +} + +.activity-tool-icon { + margin-right: 4px; +} + +.activity-tool-result .activity-tool-icon { + color: var(--success); +} + +.activity-tool-input, +.activity-tool-output { + padding: 8px 10px; + margin: 0; + font-size: 12px; + overflow-x: auto; + max-height: 200px; + overflow-y: auto; + background: var(--bg); +} + +.activity-input-bar { + display: flex; + gap: 8px; + padding: 8px 0; +} + +.activity-input-bar input { + flex: 1; + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: 6px; + color: var(--text); + font-size: 13px; +} + +.activity-input-bar input:focus { + outline: none; + border-color: var(--accent); +} + +.activity-input-bar button { + padding: 8px 16px; + background: var(--accent); + color: #fff; + border: none; + border-radius: 6px; + cursor: pointer; + font-size: 13px; +} + +.activity-input-bar button:hover { + background: var(--accent-hover); +} + +#activity-done-btn { + background: var(--bg-secondary); + border: 1px solid var(--border); + color: var(--text-secondary); +} + +#activity-done-btn:hover { + color: var(--text); + border-color: var(--text-secondary); + background: var(--bg-secondary); +} + +/* --- Copy button on code blocks --- */ + +.code-block-wrapper { + position: relative; +} + +.copy-btn { + position: absolute; + top: 6px; + right: 6px; + padding: 2px 8px; + background: var(--bg-tertiary); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text-secondary); + font-size: 11px; + cursor: pointer; + opacity: 0; + transition: opacity 0.15s; +} + +.code-block-wrapper:hover .copy-btn { + opacity: 1; +} + +.copy-btn:hover { + color: var(--text); + background: var(--border); +} + +/* --- Toast notifications --- */ + +#toasts { + position: fixed; + top: 16px; + right: 16px; + z-index: 10000; + display: flex; + flex-direction: column; + gap: 8px; + pointer-events: none; +} + +.toast { + padding: 10px 16px; + border-radius: var(--radius); + font-size: 13px; + color: #fff; + pointer-events: auto; + transform: translateX(120%); + transition: transform 0.25s ease; + max-width: 360px; + word-break: break-word; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.4); +} + +.toast.visible { + transform: translateX(0); +} + +.toast-info { + background: var(--accent); +} + +.toast-success { + background: var(--success); +} + +.toast-error { + background: var(--danger); +} + +/* --- Memory search highlighting --- */ + +mark { + background: rgba(88, 166, 255, 0.3); + color: inherit; + border-radius: 2px; + padding: 0 1px; +} + +/* --- Thread sidebar --- */ + +#tab-chat { + flex-direction: row; +} + +.thread-sidebar { + width: 200px; + background: var(--bg-secondary); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + flex-shrink: 0; + transition: width 0.2s ease; + overflow: hidden; +} + +.thread-sidebar.collapsed { + width: 36px; +} + +.thread-sidebar.collapsed .thread-sidebar-header span, +.thread-sidebar.collapsed .thread-new-btn, +.thread-sidebar.collapsed .thread-list, +.thread-sidebar.collapsed .assistant-item, +.thread-sidebar.collapsed .threads-section-header { + display: none; +} + +.thread-sidebar-header { + display: flex; + align-items: center; + padding: 10px 12px; + border-bottom: 1px solid var(--border); + font-size: 13px; + font-weight: 600; + gap: 8px; +} + +.thread-sidebar-header span { + flex: 1; +} + +.thread-new-btn { + background: none; + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--accent); + cursor: pointer; + font-size: 16px; + width: 24px; + height: 24px; + display: flex; + align-items: center; + justify-content: center; + padding: 0; + line-height: 1; +} + +.thread-new-btn:hover { + background: rgba(88, 166, 255, 0.15); +} + +.assistant-item { + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 12px; + cursor: pointer; + font-size: 13px; + font-weight: 600; + color: var(--text); + border-bottom: 1px solid var(--border); + background: var(--bg-secondary); +} + +.assistant-item:hover { + background: var(--bg-tertiary); +} + +.assistant-item.active { + background: rgba(88, 166, 255, 0.08); + color: var(--accent); + border-left: 2px solid var(--accent); +} + +.assistant-label { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.assistant-meta { + font-size: 11px; + font-weight: 400; + color: var(--text-secondary); +} + +.threads-section-header { + padding: 8px 12px 4px; + font-size: 11px; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-secondary); +} + +.thread-toggle-btn { + background: none; + border: none; + color: var(--text-secondary); + cursor: pointer; + font-size: 14px; + padding: 2px; +} + +.thread-toggle-btn:hover { + color: var(--text); +} + +.thread-list { + flex: 1; + overflow-y: auto; +} + +.thread-item { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 12px; + cursor: pointer; + font-size: 13px; + color: var(--text-secondary); + border-bottom: 1px solid rgba(255, 255, 255, 0.03); +} + +.thread-item:hover { + background: var(--bg-tertiary); + color: var(--text); +} + +.thread-item.active { + background: var(--bg-tertiary); + color: var(--accent); + border-left: 2px solid var(--accent); +} + +.thread-label { + font-family: monospace; + font-size: 12px; +} + +.thread-meta { + font-size: 11px; + color: var(--text-secondary); +} + +/* --- Memory editing --- */ + +#memory-breadcrumb-path { + flex: 1; +} + +.memory-edit-btn { + padding: 3px 10px; + background: none; + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text-secondary); + cursor: pointer; + font-size: 12px; + flex-shrink: 0; +} + +.memory-edit-btn:hover { + color: var(--accent); + border-color: var(--accent); +} + +.memory-editor { + flex: 1; + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px; + overflow: hidden; +} + +.memory-editor textarea { + flex: 1; + padding: 12px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + font-family: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace; + font-size: 13px; + line-height: 1.5; + resize: none; +} + +.memory-editor textarea:focus { + outline: none; + border-color: var(--accent); +} + +.memory-editor-actions { + display: flex; + gap: 8px; +} + +.btn-save { + padding: 6px 16px; + background: var(--accent); + color: #fff; + border: none; + border-radius: var(--radius); + cursor: pointer; + font-size: 13px; +} + +.btn-save:hover { + background: var(--accent-hover); +} + +.btn-cancel-edit { + padding: 6px 16px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + cursor: pointer; + font-size: 13px; +} + +.btn-cancel-edit:hover { + background: var(--bg-tertiary); +} + +/* Memory rendered markdown */ +.memory-viewer.rendered { + white-space: normal; + font-family: inherit; +} + +.memory-rendered { + font-size: 14px; + line-height: 1.6; +} + +.memory-rendered h1, .memory-rendered h2, .memory-rendered h3 { + margin: 12px 0 6px 0; +} + +.memory-rendered p { margin: 0 0 8px 0; } +.memory-rendered p:last-child { margin-bottom: 0; } +.memory-rendered ul, .memory-rendered ol { margin: 4px 0; padding-left: 20px; } +.memory-rendered li { margin: 2px 0; } +.memory-rendered code { + background: var(--code-bg); + padding: 1px 4px; + border-radius: 3px; + font-size: 13px; +} +.memory-rendered pre { + background: var(--code-bg); + padding: 8px 12px; + border-radius: var(--radius); + overflow-x: auto; + margin: 6px 0; +} +.memory-rendered pre code { background: none; padding: 0; } +.memory-rendered a { color: var(--accent); } +.memory-rendered blockquote { + margin: 6px 0; + padding: 4px 12px; + border-left: 3px solid var(--border); + color: var(--text-secondary); +} + +/* --- Gateway status popover --- */ + +.gateway-popover { + display: none; + position: absolute; + top: 100%; + right: 0; + margin-top: 8px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 12px; + min-width: 180px; + box-shadow: var(--shadow); + z-index: 100; +} + +.gateway-popover.visible { + display: block; +} + +.gw-stat { + display: flex; + justify-content: space-between; + font-size: 12px; + padding: 3px 0; + color: var(--text-secondary); +} + +.gw-stat span:last-child { + color: var(--text); + font-weight: 500; +} + +/* --- Extension install form --- */ + +.ext-install-form { + display: flex; + gap: 8px; + align-items: center; + flex-wrap: wrap; +} + +.ext-install-form input { + padding: 6px 10px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + font-size: 13px; +} + +.ext-install-form input:focus { + outline: none; + border-color: var(--accent); +} + +.ext-install-form select { + padding: 6px 10px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + font-size: 13px; +} + +.ext-install-form button { + padding: 6px 16px; + background: var(--accent); + color: #fff; + border: none; + border-radius: var(--radius); + cursor: pointer; + font-size: 13px; +} + +.ext-install-form button:hover { + background: var(--accent-hover); +} + +/* --- Activity toolbar --- */ + +.activity-toolbar { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 0; +} + +.activity-toolbar select { + padding: 5px 8px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--text); + font-size: 12px; +} + +.activity-toolbar select:focus { + outline: none; + border-color: var(--accent); +} + +/* --- Mobile responsive --- */ + +@media (max-width: 768px) { + /* Tab bar: horizontal scroll */ + .tab-bar { + overflow-x: auto; + -webkit-overflow-scrolling: touch; + padding: 0 8px; + } + + .tab-bar button { + padding: 8px 12px; + font-size: 13px; + white-space: nowrap; + } + + /* Chat messages: wider */ + .message { + max-width: 95%; + } + + /* Thread sidebar: hidden behind toggle */ + .thread-sidebar { + width: 36px; + } + + .thread-sidebar .thread-sidebar-header span, + .thread-sidebar .thread-new-btn, + .thread-sidebar .thread-list, + .thread-sidebar .assistant-item, + .thread-sidebar .threads-section-header { + display: none; + } + + .thread-sidebar.expanded-mobile { + position: absolute; + left: 0; + top: 0; + bottom: 0; + width: 200px; + z-index: 50; + } + + .thread-sidebar.expanded-mobile .thread-sidebar-header span, + .thread-sidebar.expanded-mobile .thread-new-btn, + .thread-sidebar.expanded-mobile .thread-list, + .thread-sidebar.expanded-mobile .assistant-item, + .thread-sidebar.expanded-mobile .threads-section-header { + display: flex; + } + + /* Memory: vertical stack */ + .memory-container { + flex-direction: column; + } + + .memory-sidebar { + width: 100%; + max-height: 200px; + border-right: none; + border-bottom: 1px solid var(--border); + } + + /* Job detail sub-tabs: wrap */ + .job-detail-tabs { + flex-wrap: wrap; + } + + .job-detail-header { + flex-wrap: wrap; + } + + .job-detail-header h2 { + min-width: 100%; + order: -1; + } + + /* Job files: vertical */ + .job-files { + flex-direction: column; + height: auto; + } + + .job-files-sidebar { + width: 100%; + max-height: 180px; + border-right: none; + border-bottom: 1px solid var(--border); + } + + /* Extension install form */ + .ext-install-form { + flex-direction: column; + align-items: stretch; + } + + .ext-install-form input, + .ext-install-form select { + width: 100%; + } +} diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index 6dc33c08..c43e86f9 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -24,10 +24,17 @@ pub struct ThreadInfo { pub turn_count: usize, pub created_at: String, pub updated_at: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_type: Option, } #[derive(Debug, Serialize)] pub struct ThreadListResponse { + /// The pinned assistant thread (always present after first load). + pub assistant_thread: Option, + /// Regular conversation threads. pub threads: Vec, pub active_thread: Option, } @@ -54,6 +61,12 @@ pub struct ToolCallInfo { pub struct HistoryResponse { pub thread_id: Uuid, pub turns: Vec, + /// Whether there are older messages available. + #[serde(default)] + pub has_more: bool, + /// Cursor for the next page (ISO8601 timestamp of the oldest message returned). + #[serde(skip_serializing_if = "Option::is_none")] + pub oldest_timestamp: Option, } // --- Approval --- @@ -63,6 +76,8 @@ pub struct ApprovalRequest { pub request_id: String, /// "approve", "always", or "deny" pub action: String, + /// Thread that owns the pending approval (so the agent loop finds the right session). + pub thread_id: Option, } // --- SSE Event Types --- @@ -73,17 +88,49 @@ pub enum SseEvent { #[serde(rename = "response")] Response { content: String, thread_id: String }, #[serde(rename = "thinking")] - Thinking { message: String }, + Thinking { + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "tool_started")] - ToolStarted { name: String }, + ToolStarted { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "tool_completed")] - ToolCompleted { name: String, success: bool }, + ToolCompleted { + name: String, + success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "tool_result")] - ToolResult { name: String, preview: String }, + ToolResult { + name: String, + preview: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "stream_chunk")] - StreamChunk { content: String }, + StreamChunk { + content: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "status")] - Status { message: String }, + Status { + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, + #[serde(rename = "job_started")] + JobStarted { + job_id: String, + title: String, + browse_url: String, + }, #[serde(rename = "approval_needed")] ApprovalNeeded { request_id: String, @@ -91,10 +138,59 @@ pub enum SseEvent { description: String, parameters: String, }, + #[serde(rename = "auth_required")] + AuthRequired { + extension_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + instructions: Option, + #[serde(skip_serializing_if = "Option::is_none")] + auth_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + setup_url: Option, + }, + #[serde(rename = "auth_completed")] + AuthCompleted { + extension_name: String, + success: bool, + message: String, + }, #[serde(rename = "error")] - Error { message: String }, + Error { + message: String, + #[serde(skip_serializing_if = "Option::is_none")] + thread_id: Option, + }, #[serde(rename = "heartbeat")] Heartbeat, + + // Sandbox job streaming events (worker + Claude Code bridge) + #[serde(rename = "job_message")] + JobMessage { + job_id: String, + role: String, + content: String, + }, + #[serde(rename = "job_tool_use")] + JobToolUse { + job_id: String, + tool_name: String, + input: serde_json::Value, + }, + #[serde(rename = "job_tool_result")] + JobToolResult { + job_id: String, + tool_name: String, + output: String, + }, + #[serde(rename = "job_status")] + JobStatus { job_id: String, message: String }, + #[serde(rename = "job_result")] + JobResult { + job_id: String, + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + session_id: Option, + }, } // --- Memory --- @@ -188,6 +284,54 @@ pub struct JobSummaryResponse { pub stuck: usize, } +#[derive(Debug, Serialize)] +pub struct JobDetailResponse { + pub id: Uuid, + pub title: String, + pub description: String, + pub state: String, + pub user_id: String, + pub created_at: String, + pub started_at: Option, + pub completed_at: Option, + pub elapsed_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub project_dir: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub browse_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub job_mode: Option, + pub transitions: Vec, +} + +// --- Project Files --- + +#[derive(Debug, Serialize)] +pub struct ProjectFileEntry { + pub name: String, + pub path: String, + pub is_dir: bool, +} + +#[derive(Debug, Serialize)] +pub struct ProjectFilesResponse { + pub entries: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ProjectFileReadResponse { + pub path: String, + pub content: String, +} + +#[derive(Debug, Serialize)] +pub struct TransitionInfo { + pub from: String, + pub to: String, + pub timestamp: String, + pub reason: Option, +} + // --- Extensions --- #[derive(Debug, Serialize)] @@ -262,6 +406,21 @@ impl ActionResponse { } } +// --- Auth Token --- + +/// Request to submit an auth token for an extension (dedicated endpoint). +#[derive(Debug, Deserialize)] +pub struct AuthTokenRequest { + pub extension_name: String, + pub token: String, +} + +/// Request to cancel an in-progress auth flow. +#[derive(Debug, Deserialize)] +pub struct AuthCancelRequest { + pub extension_name: String, +} + // --- WebSocket --- /// Message sent by a WebSocket client to the server. @@ -280,7 +439,18 @@ pub enum WsClientMessage { request_id: String, /// "approve", "always", or "deny" action: String, + /// Thread that owns the pending approval. + thread_id: Option, }, + /// Submit an auth token for an extension (bypasses message pipeline). + #[serde(rename = "auth_token")] + AuthToken { + extension_name: String, + token: String, + }, + /// Cancel an in-progress auth flow. + #[serde(rename = "auth_cancel")] + AuthCancel { extension_name: String }, /// Client heartbeat ping. #[serde(rename = "ping")] Ping, @@ -314,12 +484,20 @@ impl WsServerMessage { SseEvent::Thinking { .. } => "thinking", SseEvent::ToolStarted { .. } => "tool_started", SseEvent::ToolCompleted { .. } => "tool_completed", + SseEvent::ToolResult { .. } => "tool_result", SseEvent::StreamChunk { .. } => "stream_chunk", SseEvent::Status { .. } => "status", + SseEvent::JobStarted { .. } => "job_started", SseEvent::ApprovalNeeded { .. } => "approval_needed", + SseEvent::AuthRequired { .. } => "auth_required", + SseEvent::AuthCompleted { .. } => "auth_completed", SseEvent::Error { .. } => "error", SseEvent::Heartbeat => "heartbeat", - SseEvent::ToolResult { .. } => "tool_result", + SseEvent::JobMessage { .. } => "job_message", + SseEvent::JobToolUse { .. } => "job_tool_use", + SseEvent::JobToolResult { .. } => "job_tool_result", + SseEvent::JobStatus { .. } => "job_status", + SseEvent::JobResult { .. } => "job_result", }; let data = serde_json::to_value(event).unwrap_or(serde_json::Value::Null); WsServerMessage::Event { @@ -329,6 +507,96 @@ impl WsServerMessage { } } +// --- Routines --- + +#[derive(Debug, Serialize)] +pub struct RoutineInfo { + pub id: Uuid, + pub name: String, + pub description: String, + pub enabled: bool, + pub trigger_type: String, + pub trigger_summary: String, + pub action_type: String, + pub last_run_at: Option, + pub next_fire_at: Option, + pub run_count: u64, + pub consecutive_failures: u32, + pub status: String, +} + +#[derive(Debug, Serialize)] +pub struct RoutineListResponse { + pub routines: Vec, +} + +#[derive(Debug, Serialize)] +pub struct RoutineSummaryResponse { + pub total: u64, + pub enabled: u64, + pub disabled: u64, + pub failing: u64, + pub runs_today: u64, +} + +#[derive(Debug, Serialize)] +pub struct RoutineDetailResponse { + pub id: Uuid, + pub name: String, + pub description: String, + pub enabled: bool, + pub trigger: serde_json::Value, + pub action: serde_json::Value, + pub guardrails: serde_json::Value, + pub notify: serde_json::Value, + pub last_run_at: Option, + pub next_fire_at: Option, + pub run_count: u64, + pub consecutive_failures: u32, + pub created_at: String, + pub recent_runs: Vec, +} + +#[derive(Debug, Serialize)] +pub struct RoutineRunInfo { + pub id: Uuid, + pub trigger_type: String, + pub started_at: String, + pub completed_at: Option, + pub status: String, + pub result_summary: Option, + pub tokens_used: Option, +} + +// --- Settings --- + +#[derive(Debug, Serialize)] +pub struct SettingResponse { + pub key: String, + pub value: serde_json::Value, + pub updated_at: String, +} + +#[derive(Debug, Serialize)] +pub struct SettingsListResponse { + pub settings: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct SettingWriteRequest { + pub value: serde_json::Value, +} + +#[derive(Debug, Deserialize)] +pub struct SettingsImportRequest { + pub settings: std::collections::HashMap, +} + +#[derive(Debug, Serialize)] +pub struct SettingsExportResponse { + pub settings: std::collections::HashMap, +} + // --- Health --- #[derive(Debug, Serialize)] @@ -371,12 +639,36 @@ mod tests { #[test] fn test_ws_client_approval_parse() { - let json = r#"{"type":"approval","request_id":"abc-123","action":"approve"}"#; + let json = + r#"{"type":"approval","request_id":"abc-123","action":"approve","thread_id":"t1"}"#; let msg: WsClientMessage = serde_json::from_str(json).unwrap(); match msg { - WsClientMessage::Approval { request_id, action } => { + WsClientMessage::Approval { + request_id, + action, + thread_id, + } => { assert_eq!(request_id, "abc-123"); assert_eq!(action, "approve"); + assert_eq!(thread_id.as_deref(), Some("t1")); + } + _ => panic!("Expected Approval variant"), + } + } + + #[test] + fn test_ws_client_approval_parse_no_thread() { + let json = r#"{"type":"approval","request_id":"abc-123","action":"deny"}"#; + let msg: WsClientMessage = serde_json::from_str(json).unwrap(); + match msg { + WsClientMessage::Approval { + request_id, + action, + thread_id, + } => { + assert_eq!(request_id, "abc-123"); + assert_eq!(action, "deny"); + assert!(thread_id.is_none()); } _ => panic!("Expected Approval variant"), } @@ -437,6 +729,7 @@ mod tests { fn test_ws_server_from_sse_thinking() { let sse = SseEvent::Thinking { message: "reasoning...".to_string(), + thread_id: None, }; let ws = WsServerMessage::from_sse_event(&sse); match ws { @@ -477,4 +770,115 @@ mod tests { _ => panic!("Expected Event variant"), } } + + // ---- Auth type tests ---- + + #[test] + fn test_ws_client_auth_token_parse() { + let json = r#"{"type":"auth_token","extension_name":"notion","token":"sk-123"}"#; + let msg: WsClientMessage = serde_json::from_str(json).unwrap(); + match msg { + WsClientMessage::AuthToken { + extension_name, + token, + } => { + assert_eq!(extension_name, "notion"); + assert_eq!(token, "sk-123"); + } + _ => panic!("Expected AuthToken variant"), + } + } + + #[test] + fn test_ws_client_auth_cancel_parse() { + let json = r#"{"type":"auth_cancel","extension_name":"notion"}"#; + let msg: WsClientMessage = serde_json::from_str(json).unwrap(); + match msg { + WsClientMessage::AuthCancel { extension_name } => { + assert_eq!(extension_name, "notion"); + } + _ => panic!("Expected AuthCancel variant"), + } + } + + #[test] + fn test_sse_auth_required_serialize() { + let event = SseEvent::AuthRequired { + extension_name: "notion".to_string(), + instructions: Some("Get your token from...".to_string()), + auth_url: None, + setup_url: Some("https://notion.so/integrations".to_string()), + }; + let json = serde_json::to_string(&event).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["type"], "auth_required"); + assert_eq!(parsed["extension_name"], "notion"); + assert_eq!(parsed["instructions"], "Get your token from..."); + assert!(parsed.get("auth_url").is_none()); + assert_eq!(parsed["setup_url"], "https://notion.so/integrations"); + } + + #[test] + fn test_sse_auth_completed_serialize() { + let event = SseEvent::AuthCompleted { + extension_name: "notion".to_string(), + success: true, + message: "notion authenticated (3 tools loaded)".to_string(), + }; + let json = serde_json::to_string(&event).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed["type"], "auth_completed"); + assert_eq!(parsed["extension_name"], "notion"); + assert_eq!(parsed["success"], true); + } + + #[test] + fn test_ws_server_from_sse_auth_required() { + let sse = SseEvent::AuthRequired { + extension_name: "openai".to_string(), + instructions: Some("Enter API key".to_string()), + auth_url: None, + setup_url: None, + }; + let ws = WsServerMessage::from_sse_event(&sse); + match ws { + WsServerMessage::Event { event_type, data } => { + assert_eq!(event_type, "auth_required"); + assert_eq!(data["extension_name"], "openai"); + } + _ => panic!("Expected Event variant"), + } + } + + #[test] + fn test_ws_server_from_sse_auth_completed() { + let sse = SseEvent::AuthCompleted { + extension_name: "slack".to_string(), + success: false, + message: "Invalid token".to_string(), + }; + let ws = WsServerMessage::from_sse_event(&sse); + match ws { + WsServerMessage::Event { event_type, data } => { + assert_eq!(event_type, "auth_completed"); + assert_eq!(data["success"], false); + } + _ => panic!("Expected Event variant"), + } + } + + #[test] + fn test_auth_token_request_deserialize() { + let json = r#"{"extension_name":"telegram","token":"bot12345"}"#; + let req: AuthTokenRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extension_name, "telegram"); + assert_eq!(req.token, "bot12345"); + } + + #[test] + fn test_auth_cancel_request_deserialize() { + let json = r#"{"extension_name":"telegram"}"#; + let req: AuthCancelRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.extension_name, "telegram"); + } } diff --git a/src/channels/web/ws.rs b/src/channels/web/ws.rs index 64755e3c..d6ebc0f0 100644 --- a/src/channels/web/ws.rs +++ b/src/channels/web/ws.rs @@ -71,8 +71,17 @@ pub async fn handle_ws_connection(socket: WebSocket, state: Arc) { } let tracker_for_drop = state.ws_tracker.clone(); - // Subscribe to broadcast events (same source as SSE) - let mut event_stream = Box::pin(state.sse.subscribe_raw()); + // Subscribe to broadcast events (same source as SSE). + // Reject if we've hit the connection limit. + let Some(raw_stream) = state.sse.subscribe_raw() else { + tracing::warn!("WebSocket rejected: too many connections"); + // Decrement the WS tracker we already incremented above. + if let Some(ref tracker) = tracker_for_drop { + tracker.decrement(); + } + return; + }; + let mut event_stream = Box::pin(raw_stream); // Channel for the sender task to receive messages from both // the broadcast stream and any direct sends (like Pong) @@ -170,7 +179,11 @@ async fn handle_client_message( .await; } } - WsClientMessage::Approval { request_id, action } => { + WsClientMessage::Approval { + request_id, + action, + thread_id, + } => { let (approved, always) = match action.as_str() { "approve" => (true, false), "always" => (true, true), @@ -214,12 +227,71 @@ async fn handle_client_message( } }; - let msg = IncomingMessage::new("gateway", user_id, content); + let mut msg = IncomingMessage::new("gateway", user_id, content); + if let Some(ref tid) = thread_id { + msg = msg.with_thread(tid); + } let tx_guard = state.msg_tx.read().await; if let Some(ref tx) = *tx_guard { let _ = tx.send(msg).await; } } + WsClientMessage::AuthToken { + extension_name, + token, + } => { + if let Some(ref ext_mgr) = state.extension_manager { + match ext_mgr.auth(&extension_name, Some(&token)).await { + Ok(result) if result.status == "authenticated" => { + let msg = match ext_mgr.activate(&extension_name).await { + Ok(r) => format!( + "{} authenticated ({} tools loaded)", + extension_name, + r.tools_loaded.len() + ), + Err(e) => format!( + "{} authenticated but activation failed: {}", + extension_name, e + ), + }; + crate::channels::web::server::clear_auth_mode(state).await; + state + .sse + .broadcast(crate::channels::web::types::SseEvent::AuthCompleted { + extension_name, + success: true, + message: msg, + }); + } + Ok(result) => { + state + .sse + .broadcast(crate::channels::web::types::SseEvent::AuthRequired { + extension_name, + instructions: result.instructions, + auth_url: result.auth_url, + setup_url: result.setup_url, + }); + } + Err(e) => { + let _ = direct_tx + .send(WsServerMessage::Error { + message: format!("Auth failed: {}", e), + }) + .await; + } + } + } else { + let _ = direct_tx + .send(WsServerMessage::Error { + message: "Extension manager not available".to_string(), + }) + .await; + } + } + WsClientMessage::AuthCancel { .. } => { + crate::channels::web::server::clear_auth_mode(state).await; + } WsClientMessage::Ping => { let _ = direct_tx.send(WsServerMessage::Pong).await; } @@ -328,6 +400,7 @@ mod tests { WsClientMessage::Approval { request_id: request_id.to_string(), action: "approve".to_string(), + thread_id: Some("thread-42".to_string()), }, &state, "user1", @@ -338,6 +411,8 @@ mod tests { let incoming = agent_rx.recv().await.unwrap(); // The content should be a serialized ExecApproval assert!(incoming.content.contains("ExecApproval")); + // Thread should be forwarded onto the IncomingMessage. + assert_eq!(incoming.thread_id.as_deref(), Some("thread-42")); } #[tokio::test] @@ -349,6 +424,7 @@ mod tests { WsClientMessage::Approval { request_id: Uuid::new_v4().to_string(), action: "maybe".to_string(), + thread_id: None, }, &state, "user1", @@ -374,6 +450,7 @@ mod tests { WsClientMessage::Approval { request_id: "not-a-uuid".to_string(), action: "approve".to_string(), + thread_id: None, }, &state, "user1", @@ -398,14 +475,18 @@ mod tests { msg_tx: tokio::sync::RwLock::new(msg_tx), sse: SseManager::new(), workspace: None, - context_manager: None, session_manager: None, log_broadcaster: None, extension_manager: None, tool_registry: None, + store: None, + job_manager: None, + prompt_queue: None, user_id: "test".to_string(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: Some(Arc::new(WsConnectionTracker::new())), + llm_provider: None, + chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60), } } } diff --git a/src/cli/config.rs b/src/cli/config.rs index 080cccf1..f91e9241 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -1,6 +1,9 @@ //! Configuration management CLI commands. //! //! Commands for viewing and modifying settings. +//! Settings are stored in the database (env > DB > default). + +use std::sync::Arc; use clap::Subcommand; @@ -36,41 +39,81 @@ pub enum ConfigCommand { path: String, }, - /// Show the settings file path + /// Show the settings storage info Path, } /// Run a config command. -pub fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> { +/// +/// Connects to the database to read/write settings. Falls back to disk +/// if the database is not available. +pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> { + // Try to connect to the DB for settings access + let db: Option> = match connect_db().await { + Ok(d) => Some(d), + Err(e) => { + eprintln!( + "Warning: Could not connect to database ({}), using disk fallback", + e + ); + None + } + }; + + let db_ref = db.as_deref(); match cmd { - ConfigCommand::List { filter } => list_settings(filter), - ConfigCommand::Get { path } => get_setting(&path), - ConfigCommand::Set { path, value } => set_setting(&path, &value), - ConfigCommand::Reset { path } => reset_setting(&path), - ConfigCommand::Path => show_path(), + ConfigCommand::List { filter } => list_settings(db_ref, filter).await, + ConfigCommand::Get { path } => get_setting(db_ref, &path).await, + ConfigCommand::Set { path, value } => set_setting(db_ref, &path, &value).await, + ConfigCommand::Reset { path } => reset_setting(db_ref, &path).await, + ConfigCommand::Path => show_path(db_ref.is_some()), } } +/// Bootstrap a DB connection for config commands (backend-agnostic). +async fn connect_db() -> anyhow::Result> { + let config = crate::config::Config::from_env() + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + crate::db::connect_from_config(&config.database) + .await + .map_err(|e| anyhow::anyhow!("{}", e)) +} + +const DEFAULT_USER_ID: &str = "default"; + +/// Load settings: DB if available, else disk. +async fn load_settings(store: Option<&dyn crate::db::Database>) -> Settings { + if let Some(store) = store { + match store.get_all_settings(DEFAULT_USER_ID).await { + Ok(map) if !map.is_empty() => return Settings::from_db_map(&map), + _ => {} + } + } + Settings::default() +} + /// List all settings. -fn list_settings(filter: Option) -> anyhow::Result<()> { - let settings = Settings::load(); +async fn list_settings( + store: Option<&dyn crate::db::Database>, + filter: Option, +) -> anyhow::Result<()> { + let settings = load_settings(store).await; let all = settings.list(); - // Find the longest key for alignment let max_key_len = all.iter().map(|(k, _)| k.len()).max().unwrap_or(0); - println!("Settings:"); + let source = if store.is_some() { "database" } else { "disk" }; + println!("Settings (source: {}):", source); println!(); for (key, value) in all { - // Skip if filter is set and doesn't match - if let Some(ref f) = filter { - if !key.starts_with(f) { - continue; - } + if let Some(ref f) = filter + && !key.starts_with(f) + { + continue; } - // Truncate long values for display let display_value = if value.len() > 60 { format!("{}...", &value[..57]) } else { @@ -84,8 +127,8 @@ fn list_settings(filter: Option) -> anyhow::Result<()> { } /// Get a specific setting. -fn get_setting(path: &str) -> anyhow::Result<()> { - let settings = Settings::load(); +async fn get_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> { + let settings = load_settings(store).await; match settings.get(path) { Some(value) => { @@ -99,68 +142,63 @@ fn get_setting(path: &str) -> anyhow::Result<()> { } /// Set a setting value. -fn set_setting(path: &str, value: &str) -> anyhow::Result<()> { - let mut settings = Settings::load(); +async fn set_setting( + store: Option<&dyn crate::db::Database>, + path: &str, + value: &str, +) -> anyhow::Result<()> { + let mut settings = load_settings(store).await; - // Try to set the value settings .set(path, value) .map_err(|e| anyhow::anyhow!("{}", e))?; - // Save to disk - settings.save()?; + let store = store.ok_or_else(|| { + anyhow::anyhow!("Database connection required to save settings. Check DATABASE_URL.") + })?; + let json_value = match serde_json::from_str::(value) { + Ok(v) => v, + Err(_) => serde_json::Value::String(value.to_string()), + }; + store + .set_setting(DEFAULT_USER_ID, path, &json_value) + .await + .map_err(|e| anyhow::anyhow!("Failed to save to database: {}", e))?; println!("Set {} = {}", path, value); Ok(()) } /// Reset a setting to default. -fn reset_setting(path: &str) -> anyhow::Result<()> { - let mut settings = Settings::load(); - - // Get the default value for display +async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> { let default = Settings::default(); let default_value = default .get(path) .ok_or_else(|| anyhow::anyhow!("Unknown setting: {}", path))?; - // Reset it - settings.reset(path).map_err(|e| anyhow::anyhow!("{}", e))?; - - // Save to disk - settings.save()?; + let store = store.ok_or_else(|| { + anyhow::anyhow!("Database connection required to reset settings. Check DATABASE_URL.") + })?; + store + .delete_setting(DEFAULT_USER_ID, path) + .await + .map_err(|e| anyhow::anyhow!("Failed to delete setting from database: {}", e))?; println!("Reset {} to default: {}", path, default_value); Ok(()) } -/// Show the settings file path. -fn show_path() -> anyhow::Result<()> { - let path = Settings::default_path(); - println!("{}", path.display()); - - if path.exists() { - let metadata = std::fs::metadata(&path)?; - println!(" Size: {} bytes", metadata.len()); - if let Ok(modified) = metadata.modified() { - use std::time::SystemTime; - let duration = SystemTime::now() - .duration_since(modified) - .unwrap_or_default(); - let secs = duration.as_secs(); - if secs < 60 { - println!(" Modified: {} seconds ago", secs); - } else if secs < 3600 { - println!(" Modified: {} minutes ago", secs / 60); - } else if secs < 86400 { - println!(" Modified: {} hours ago", secs / 3600); - } else { - println!(" Modified: {} days ago", secs / 86400); - } - } +/// Show the settings storage info. +fn show_path(has_db: bool) -> anyhow::Result<()> { + if has_db { + println!("Settings stored in: database (settings table)"); } else { - println!(" (does not exist, using defaults)"); + println!("Settings stored in: PostgreSQL (not connected, using defaults)"); } + println!( + "Env config: {}", + crate::bootstrap::ironclaw_env_path().display() + ); Ok(()) } diff --git a/src/cli/mcp.rs b/src/cli/mcp.rs index db0c65be..e61b65de 100644 --- a/src/cli/mcp.rs +++ b/src/cli/mcp.rs @@ -8,14 +8,14 @@ use std::sync::Arc; use clap::Subcommand; use crate::config::Config; -use crate::history::Store; -use crate::secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore}; +use crate::db::Database; +#[cfg(feature = "postgres")] +use crate::secrets::PostgresSecretsStore; +use crate::secrets::{SecretsCrypto, SecretsStore}; use crate::tools::mcp::{ McpClient, McpServerConfig, McpSessionManager, OAuthConfig, auth::{authorize_mcp_server, is_authenticated}, - config::{ - add_mcp_server, get_mcp_server, load_mcp_servers, remove_mcp_server, save_mcp_servers, - }, + config::{self, McpServersFile}, }; #[derive(Subcommand, Debug, Clone)] @@ -173,8 +173,11 @@ async fn add_server( // Validate config.validate()?; - // Save - add_mcp_server(config).await?; + // Save (DB if available, else disk) + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; + servers.upsert(config); + save_servers(db.as_deref(), &servers).await?; println!(); println!(" ✓ Added MCP server '{}'", name); @@ -192,7 +195,12 @@ async fn add_server( /// Remove an MCP server. async fn remove_server(name: String) -> anyhow::Result<()> { - remove_mcp_server(&name).await?; + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; + if !servers.remove(&name) { + anyhow::bail!("Server '{}' not found", name); + } + save_servers(db.as_deref(), &servers).await?; println!(); println!(" ✓ Removed MCP server '{}'", name); @@ -203,7 +211,8 @@ async fn remove_server(name: String) -> anyhow::Result<()> { /// List configured MCP servers. async fn list_servers(verbose: bool) -> anyhow::Result<()> { - let servers = load_mcp_servers().await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; if servers.servers.is_empty() { println!(); @@ -261,7 +270,12 @@ async fn list_servers(verbose: bool) -> anyhow::Result<()> { /// Authenticate with an MCP server. async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> { // Get server config - let server = get_mcp_server(&name).await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; + let server = servers + .get(&name) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Server '{}' not found", name))?; // Initialize secrets store let secrets = get_secrets_store().await?; @@ -329,7 +343,12 @@ async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> { /// Test connection to an MCP server. async fn test_server(name: String, user_id: String) -> anyhow::Result<()> { // Get server config - let server = get_mcp_server(&name).await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; + let server = servers + .get(&name) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Server '{}' not found", name))?; println!(); println!(" Testing connection to '{}'...", name); @@ -420,7 +439,8 @@ async fn test_server(name: String, user_id: String) -> anyhow::Result<()> { /// Toggle server enabled/disabled state. async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Result<()> { - let mut servers = load_mcp_servers().await?; + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; let server = servers .get_mut(&name) @@ -435,7 +455,7 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res }; server.enabled = new_state; - save_mcp_servers(&servers).await?; + save_servers(db.as_deref(), &servers).await?; let status = if new_state { "enabled" } else { "disabled" }; println!(); @@ -445,9 +465,38 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res Ok(()) } +const DEFAULT_USER_ID: &str = "default"; + +/// Try to connect to the database (backend-agnostic). +async fn connect_db() -> Option> { + let config = Config::from_env().await.ok()?; + crate::db::connect_from_config(&config.database).await.ok() +} + +/// Load MCP servers (DB if available, else disk). +async fn load_servers(db: Option<&dyn Database>) -> Result { + if let Some(db) = db { + config::load_mcp_servers_from_db(db, DEFAULT_USER_ID).await + } else { + config::load_mcp_servers().await + } +} + +/// Save MCP servers (DB if available, else disk). +async fn save_servers( + db: Option<&dyn Database>, + servers: &McpServersFile, +) -> Result<(), config::ConfigError> { + if let Some(db) = db { + config::save_mcp_servers_to_db(db, DEFAULT_USER_ID, servers).await + } else { + config::save_mcp_servers(servers).await + } +} + /// Initialize and return the secrets store. async fn get_secrets_store() -> anyhow::Result> { - let config = Config::from_env()?; + let config = Config::from_env().await?; let master_key = config.secrets.master_key().ok_or_else(|| { anyhow::anyhow!( @@ -455,14 +504,61 @@ async fn get_secrets_store() -> anyhow::Result, + embeddings: Option>, +) -> anyhow::Result<()> { + let mut workspace = Workspace::new_with_db("default", db); + if let Some(emb) = embeddings { + workspace = workspace.with_embeddings(emb); + } + + match cmd { + MemoryCommand::Search { query, limit } => search(&workspace, &query, limit).await, + MemoryCommand::Read { path } => read(&workspace, &path).await, + MemoryCommand::Write { + path, + content, + append, + } => write(&workspace, &path, content, append).await, + MemoryCommand::Tree { path, depth } => tree(&workspace, &path, depth).await, + MemoryCommand::Status => status(&workspace).await, + } +} + #[derive(Subcommand, Debug, Clone)] pub enum MemoryCommand { /// Search workspace memory (hybrid full-text + semantic) @@ -55,7 +79,8 @@ pub enum MemoryCommand { Status, } -/// Run a memory command. +/// Run a memory command (PostgreSQL backend). +#[cfg(feature = "postgres")] pub async fn run_memory_command( cmd: MemoryCommand, pool: deadpool_postgres::Pool, diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 005a22c4..06715d8c 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -12,12 +12,18 @@ mod config; mod mcp; pub mod memory; +pub mod oauth_defaults; +mod pairing; pub mod status; mod tool; pub use config::{ConfigCommand, run_config_command}; pub use mcp::{McpCommand, run_mcp_command}; -pub use memory::{MemoryCommand, run_memory_command}; +pub use memory::MemoryCommand; +#[cfg(feature = "postgres")] +pub use memory::run_memory_command; +pub use memory::run_memory_command_with_db; +pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store}; pub use status::run_status_command; pub use tool::{ToolCommand, run_tool_command}; @@ -86,8 +92,48 @@ pub enum Command { #[command(subcommand)] Memory(MemoryCommand), + /// DM pairing (approve inbound requests from unknown senders) + #[command(subcommand)] + Pairing(PairingCommand), + /// Show system health and diagnostics Status, + + /// Run as a sandboxed worker inside a Docker container (internal use). + /// This is invoked automatically by the orchestrator, not by users directly. + Worker { + /// Job ID to execute. + #[arg(long)] + job_id: uuid::Uuid, + + /// URL of the orchestrator's internal API. + #[arg(long, default_value = "http://host.docker.internal:50051")] + orchestrator_url: String, + + /// Maximum iterations before stopping. + #[arg(long, default_value = "50")] + max_iterations: u32, + }, + + /// Run as a Claude Code bridge inside a Docker container (internal use). + /// Spawns the `claude` CLI and streams output back to the orchestrator. + ClaudeBridge { + /// Job ID to execute. + #[arg(long)] + job_id: uuid::Uuid, + + /// URL of the orchestrator's internal API. + #[arg(long, default_value = "http://host.docker.internal:50051")] + orchestrator_url: String, + + /// Maximum agentic turns for Claude Code. + #[arg(long, default_value = "50")] + max_turns: u32, + + /// Claude model to use (e.g. "sonnet", "opus"). + #[arg(long, default_value = "sonnet")] + model: String, + }, } impl Cli { diff --git a/src/cli/oauth_defaults.rs b/src/cli/oauth_defaults.rs new file mode 100644 index 00000000..eea91a71 --- /dev/null +++ b/src/cli/oauth_defaults.rs @@ -0,0 +1,343 @@ +//! Shared OAuth infrastructure: built-in credentials, callback server, landing pages. +//! +//! Every OAuth flow in the codebase (WASM tool auth, MCP server auth, NEAR AI login) +//! uses the same callback port, landing page, and listener logic from this module. +//! +//! # Built-in Credentials +//! +//! Many CLI tools (gcloud, rclone, gdrive) ship with default OAuth credentials +//! so users don't need to register their own OAuth app. Google explicitly +//! documents that client_secret for "Desktop App" / "Installed App" types +//! is NOT actually secret. +//! +//! Default credentials are hardcoded below. They can be overridden at: +//! +//! - **Compile time**: Set IRONCLAW_GOOGLE_CLIENT_ID / IRONCLAW_GOOGLE_CLIENT_SECRET +//! env vars before building to replace the hardcoded defaults. +//! - **Runtime**: Users can set GOOGLE_OAUTH_CLIENT_ID / GOOGLE_OAUTH_CLIENT_SECRET +//! env vars, which take priority over built-in defaults. + +use std::time::Duration; + +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpListener; + +// ── Built-in credentials ──────────────────────────────────────────────── + +pub struct OAuthCredentials { + pub client_id: &'static str, + pub client_secret: &'static str, +} + +/// Google OAuth "Desktop App" credentials, shared across all Google tools. +/// Compile-time env vars override the hardcoded defaults below. +const GOOGLE_CLIENT_ID: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_ID") { + Some(v) => v, + None => "564604149681-efo25d43rs85v0tibdepsmdv5dsrhhr0.apps.googleusercontent.com", +}; +const GOOGLE_CLIENT_SECRET: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_SECRET") { + Some(v) => v, + None => "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", +}; + +/// Returns built-in OAuth credentials for a provider, keyed by secret_name. +/// +/// The secret_name comes from the tool's capabilities.json `auth.secret_name` field. +/// Returns `None` if no built-in credentials are configured for that provider. +pub fn builtin_credentials(secret_name: &str) -> Option { + match secret_name { + "google_oauth_token" => Some(OAuthCredentials { + client_id: GOOGLE_CLIENT_ID, + client_secret: GOOGLE_CLIENT_SECRET, + }), + _ => None, + } +} + +// ── Shared callback server ────────────────────────────────────────────── + +/// Fixed port for all OAuth callbacks. +/// +/// Every redirect URI registered with providers must use this port: +/// `http://localhost:9876/callback` (or `/auth/callback` for NEAR AI). +pub const OAUTH_CALLBACK_PORT: u16 = 9876; + +/// Error from the OAuth callback listener. +#[derive(Debug, thiserror::Error)] +pub enum OAuthCallbackError { + #[error("Port {0} is in use (another auth flow running?): {1}")] + PortInUse(u16, String), + + #[error("Authorization denied by user")] + Denied, + + #[error("Timed out waiting for authorization")] + Timeout, + + #[error("IO error: {0}")] + Io(String), +} + +/// Bind the OAuth callback listener on the fixed port. +/// +/// Tries IPv6 loopback (`[::1]`) first so that `http://localhost:…` redirects +/// work on systems where `localhost` resolves to `::1`. Falls back to IPv4 +/// (`127.0.0.1`) only if IPv6 fails for a reason other than `AddrInUse` +/// (e.g., IPv6 not supported on the host). If the port is already occupied +/// on IPv6, the port is occupied period, so we fail immediately. +pub async fn bind_callback_listener() -> Result { + let ipv6_addr = format!("[::1]:{}", OAUTH_CALLBACK_PORT); + match TcpListener::bind(&ipv6_addr).await { + Ok(listener) => return Ok(listener), + Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => { + return Err(OAuthCallbackError::PortInUse( + OAUTH_CALLBACK_PORT, + e.to_string(), + )); + } + Err(_) => { + // IPv6 not available on this host, fall back to IPv4 + } + } + TcpListener::bind(format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT)) + .await + .map_err(|e| { + if e.kind() == std::io::ErrorKind::AddrInUse { + OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string()) + } else { + OAuthCallbackError::Io(e.to_string()) + } + }) +} + +/// Wait for an OAuth callback and extract a query parameter value. +/// +/// Listens for a GET request matching `path_prefix` (e.g., "/callback" or "/auth/callback"), +/// extracts the value of `param_name` (e.g., "code" or "token"), and shows a branded +/// landing page using `display_name` (e.g., "Google", "Notion", "NEAR AI"). +/// +/// Times out after 5 minutes. +pub async fn wait_for_callback( + listener: TcpListener, + path_prefix: &str, + param_name: &str, + display_name: &str, +) -> Result { + let path_prefix = path_prefix.to_string(); + let param_name = param_name.to_string(); + let display_name = display_name.to_string(); + + tokio::time::timeout(Duration::from_secs(300), async move { + loop { + let (mut socket, _) = listener + .accept() + .await + .map_err(|e| OAuthCallbackError::Io(e.to_string()))?; + + let mut reader = BufReader::new(&mut socket); + let mut request_line = String::new(); + reader + .read_line(&mut request_line) + .await + .map_err(|e| OAuthCallbackError::Io(e.to_string()))?; + + if let Some(path) = request_line.split_whitespace().nth(1) + && path.starts_with(&path_prefix) + && let Some(query) = path.split('?').nth(1) + { + // Check for error first + if query.contains("error=") { + let html = landing_html(&display_name, false); + let response = format!( + "HTTP/1.1 400 Bad Request\r\n\ + Content-Type: text/html; charset=utf-8\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + html + ); + let _ = socket.write_all(response.as_bytes()).await; + return Err(OAuthCallbackError::Denied); + } + + // Look for the target parameter + for param in query.split('&') { + let parts: Vec<&str> = param.splitn(2, '=').collect(); + if parts.len() == 2 && parts[0] == param_name { + let value = urlencoding::decode(parts[1]) + .unwrap_or_else(|_| parts[1].into()) + .into_owned(); + + let html = landing_html(&display_name, true); + let response = format!( + "HTTP/1.1 200 OK\r\n\ + Content-Type: text/html; charset=utf-8\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + html + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + + return Ok(value); + } + } + } + + // Not the callback we're looking for + let response = "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(response.as_bytes()).await; + } + }) + .await + .map_err(|_| OAuthCallbackError::Timeout)? +} + +/// Escape a string for safe interpolation into HTML content. +fn html_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(c), + } + } + out +} + +/// HTML landing page shown in the browser after an OAuth redirect. +pub fn landing_html(provider_name: &str, success: bool) -> String { + let safe_name = html_escape(provider_name); + let (icon, heading, subtitle, accent) = if success { + ( + r##"
+ +
"##, + format!("{} Connected", safe_name), + "You can close this window and return to your terminal.", + "#22c55e", + ) + } else { + ( + r##"
+ +
"##, + "Authorization Failed".to_string(), + "The request was denied. You can close this window and try again.", + "#ef4444", + ) + }; + + format!( + r#" + + + + +IronClaw - {heading} + + + +
+ {icon} +

{heading}

+

{subtitle}

+
IronClaw
+
+ +"#, + heading = heading, + icon = icon, + subtitle = subtitle, + accent = accent, + ) +} + +#[cfg(test)] +mod tests { + use crate::cli::oauth_defaults::{builtin_credentials, landing_html}; + + #[test] + fn test_unknown_provider_returns_none() { + assert!(builtin_credentials("unknown_token").is_none()); + } + + #[test] + fn test_google_returns_based_on_compile_env() { + let creds = builtin_credentials("google_oauth_token"); + assert!(creds.is_some()); + let creds = creds.unwrap(); + assert!(!creds.client_id.is_empty()); + assert!(!creds.client_secret.is_empty()); + } + + #[test] + fn test_landing_html_success_contains_key_elements() { + let html = landing_html("Google", true); + assert!(html.contains("Google Connected")); + assert!(html.contains("charset")); + assert!(html.contains("IronClaw")); + assert!(html.contains("#22c55e")); // green accent + assert!(!html.contains("Failed")); + } + + #[test] + fn test_landing_html_escapes_provider_name() { + let html = landing_html("", true); + assert!(!html.contains("