mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0f347d1f63 | ||
|
|
3829d81269 | ||
|
|
8a4f3b6f88 | ||
|
|
140f29decf | ||
|
|
5725a62c83 | ||
|
|
bfe393eb38 | ||
|
|
9906190de7 | ||
|
|
9349a3baca | ||
|
|
3f135bdde9 | ||
|
|
97a7637f30 | ||
|
|
dae26d640e | ||
|
|
fa64df05ff | ||
|
|
356f56f77c |
+23
-8
@@ -2,18 +2,27 @@
|
||||
DATABASE_URL=postgres://localhost/ironclaw
|
||||
DATABASE_POOL_SIZE=10
|
||||
|
||||
# LLM Provider (NEAR AI)
|
||||
# NEAR AI provides a unified interface to all models with user authentication
|
||||
# Session token is stored in ~/.ironclaw/session.json and managed automatically.
|
||||
# On first run, the agent will open a browser for OAuth authentication.
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
# LLM Provider
|
||||
# LLM_BACKEND=nearai # default
|
||||
# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil
|
||||
|
||||
# === NEAR AI Chat (Responses API, session token auth) ===
|
||||
# Default mode. Uses browser OAuth (GitHub/Google) on first run.
|
||||
# Session token stored in ~/.ironclaw/session.json automatically.
|
||||
# For hosting providers: set NEARAI_SESSION_TOKEN env var directly.
|
||||
NEARAI_MODEL=zai-org/GLM-5-FP8
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
NEARAI_AUTH_URL=https://private.near.ai
|
||||
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
|
||||
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
|
||||
|
||||
# === NEAR AI Cloud (Chat Completions API, API key auth) ===
|
||||
# Auto-selected when NEARAI_API_KEY is set. Get a key from cloud.near.ai.
|
||||
# NEARAI_API_KEY=...
|
||||
# NEARAI_BASE_URL=https://cloud-api.near.ai # default for cloud mode
|
||||
# NEARAI_API_MODE=chat_completions # auto-detected from API key
|
||||
|
||||
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
|
||||
# LLM_BACKEND=nearai # default
|
||||
# Possible values: nearai, ollama, openai_compatible, openai, anthropic
|
||||
|
||||
# === Ollama ===
|
||||
# OLLAMA_MODEL=llama3.2
|
||||
@@ -68,6 +77,12 @@ HEARTBEAT_INTERVAL_SECS=1800
|
||||
HEARTBEAT_NOTIFY_CHANNEL=cli
|
||||
HEARTBEAT_NOTIFY_USER=default
|
||||
|
||||
# Memory hygiene settings (automatic cleanup of stale workspace documents)
|
||||
# Runs on each heartbeat tick; identity files (IDENTITY.md, SOUL.md) are never deleted
|
||||
# MEMORY_HYGIENE_ENABLED=true
|
||||
# MEMORY_HYGIENE_RETENTION_DAYS=30 # delete daily/ docs older than this many days
|
||||
# MEMORY_HYGIENE_CADENCE_HOURS=12 # minimum hours between cleanup passes
|
||||
|
||||
# Safety settings
|
||||
SAFETY_MAX_OUTPUT_LENGTH=100000
|
||||
SAFETY_INJECTION_CHECK_ENABLED=true
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# Scope labels for actions/labeler@v6
|
||||
# Maps file path globs to scope labels. Multiple labels can apply per PR.
|
||||
|
||||
"scope: agent":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/agent/**
|
||||
|
||||
"scope: channel":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/channel.rs
|
||||
- src/channels/manager.rs
|
||||
- src/channels/mod.rs
|
||||
|
||||
"scope: channel/cli":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/cli/**
|
||||
- src/cli/**
|
||||
|
||||
"scope: channel/web":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/web/**
|
||||
|
||||
"scope: channel/wasm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/wasm/**
|
||||
|
||||
"scope: tool":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/tool.rs
|
||||
- src/tools/registry.rs
|
||||
- src/tools/mod.rs
|
||||
- src/tools/sandbox.rs
|
||||
|
||||
"scope: tool/builtin":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/builtin/**
|
||||
|
||||
"scope: tool/wasm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/wasm/**
|
||||
|
||||
"scope: tool/mcp":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/mcp/**
|
||||
|
||||
"scope: tool/builder":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/builder/**
|
||||
|
||||
"scope: db":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/mod.rs
|
||||
|
||||
"scope: db/postgres":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/postgres.rs
|
||||
- migrations/**
|
||||
|
||||
"scope: db/libsql":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/libsql_backend.rs
|
||||
- src/db/libsql_migrations.rs
|
||||
|
||||
"scope: safety":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/safety/**
|
||||
|
||||
"scope: llm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/llm/**
|
||||
|
||||
"scope: workspace":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/workspace/**
|
||||
|
||||
"scope: orchestrator":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/orchestrator/**
|
||||
|
||||
"scope: worker":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/worker/**
|
||||
|
||||
"scope: secrets":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/secrets/**
|
||||
|
||||
"scope: config":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/config.rs
|
||||
- src/settings.rs
|
||||
|
||||
"scope: extensions":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/extensions/**
|
||||
|
||||
"scope: setup":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/setup/**
|
||||
|
||||
"scope: evaluation":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/evaluation/**
|
||||
|
||||
"scope: estimation":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/estimation/**
|
||||
|
||||
"scope: sandbox":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/sandbox/**
|
||||
- Dockerfile*
|
||||
|
||||
"scope: hooks":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/hooks/**
|
||||
|
||||
"scope: pairing":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/pairing/**
|
||||
|
||||
"scope: ci":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- .github/workflows/**
|
||||
- .github/scripts/**
|
||||
|
||||
"scope: docs":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "**/*.md"
|
||||
- docs/**
|
||||
- LICENSE*
|
||||
|
||||
"scope: dependencies":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Idempotent label bootstrap for IronClaw PR automation.
|
||||
# Uses `gh label create --force` so it can be re-run safely.
|
||||
#
|
||||
# Usage: bash .github/scripts/create-labels.sh
|
||||
# Requires: gh CLI authenticated with repo scope
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v gh &>/dev/null; then
|
||||
echo "Error: gh CLI is required. Install from https://cli.github.com" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create() {
|
||||
local name="$1" color="$2" description="$3"
|
||||
gh label create "$name" --color "$color" --description "$description" --force
|
||||
}
|
||||
|
||||
echo "==> Creating size labels..."
|
||||
create "size: XS" "F9D0C4" "< 10 changed lines (excluding docs)"
|
||||
create "size: S" "F5A3A3" "10-49 changed lines"
|
||||
create "size: M" "E57373" "50-199 changed lines"
|
||||
create "size: L" "D32F2F" "200-499 changed lines"
|
||||
create "size: XL" "B71C1C" "500+ changed lines"
|
||||
|
||||
echo "==> Creating risk labels..."
|
||||
create "risk: low" "4CAF50" "Changes to docs, tests, or low-risk modules"
|
||||
create "risk: medium" "FFC107" "Business logic, config, or moderate-risk modules"
|
||||
create "risk: high" "F44336" "Safety, secrets, auth, or critical infrastructure"
|
||||
create "risk: manual" "9E9E9E" "Risk level set manually (sticky, not overwritten)"
|
||||
|
||||
echo "==> Creating scope labels..."
|
||||
create "scope: agent" "006B75" "Agent core (agent loop, router, scheduler)"
|
||||
create "scope: channel" "00838F" "Channel infrastructure"
|
||||
create "scope: channel/cli" "00897B" "TUI / CLI channel"
|
||||
create "scope: channel/web" "00796B" "Web gateway channel"
|
||||
create "scope: channel/wasm" "00695C" "WASM channel runtime"
|
||||
create "scope: tool" "1565C0" "Tool infrastructure"
|
||||
create "scope: tool/builtin" "1976D2" "Built-in tools"
|
||||
create "scope: tool/wasm" "1E88E5" "WASM tool sandbox"
|
||||
create "scope: tool/mcp" "2196F3" "MCP client"
|
||||
create "scope: tool/builder" "42A5F5" "Dynamic tool builder"
|
||||
create "scope: db" "4A148C" "Database trait / abstraction"
|
||||
create "scope: db/postgres" "6A1B9A" "PostgreSQL backend"
|
||||
create "scope: db/libsql" "7B1FA2" "libSQL / Turso backend"
|
||||
create "scope: safety" "880E4F" "Prompt injection defense"
|
||||
create "scope: llm" "4527A0" "LLM integration"
|
||||
create "scope: workspace" "283593" "Persistent memory / workspace"
|
||||
create "scope: orchestrator" "0D47A1" "Container orchestrator"
|
||||
create "scope: worker" "01579B" "Container worker"
|
||||
create "scope: secrets" "BF360C" "Secrets management"
|
||||
create "scope: config" "E65100" "Configuration"
|
||||
create "scope: extensions" "33691E" "Extension management"
|
||||
create "scope: setup" "827717" "Onboarding / setup"
|
||||
create "scope: evaluation" "558B2F" "Success evaluation"
|
||||
create "scope: estimation" "9E9D24" "Cost/time estimation"
|
||||
create "scope: sandbox" "00BFA5" "Docker sandbox"
|
||||
create "scope: hooks" "6D4C41" "Git/event hooks"
|
||||
create "scope: pairing" "4E342E" "Pairing mode"
|
||||
create "scope: ci" "546E7A" "CI/CD workflows"
|
||||
create "scope: docs" "78909C" "Documentation"
|
||||
create "scope: dependencies" "90A4AE" "Dependency updates"
|
||||
|
||||
echo "==> Creating contributor labels..."
|
||||
create "contributor: new" "FFF9C4" "First-time contributor"
|
||||
create "contributor: regular" "FFE082" "2-5 merged PRs"
|
||||
create "contributor: experienced" "FFB74D" "6-19 merged PRs"
|
||||
create "contributor: core" "FF8A65" "20+ merged PRs"
|
||||
|
||||
echo "Done. All labels created/updated."
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# Classify a PR by size, risk, and contributor tier.
|
||||
# Called by the pr-label-classify workflow.
|
||||
#
|
||||
# Inputs (env vars):
|
||||
# PR_NUMBER — pull request number
|
||||
# REPO — owner/repo (e.g. "user/ironclaw")
|
||||
#
|
||||
# Requires: gh CLI, jq
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PR_NUMBER="${PR_NUMBER:?PR_NUMBER is required}"
|
||||
REPO="${REPO:?REPO is required}"
|
||||
|
||||
# ─── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Remove all labels in a dimension except the desired one.
|
||||
# Usage: set_exclusive_label "size" "size: M"
|
||||
set_exclusive_label() {
|
||||
local prefix="$1" desired="$2"
|
||||
|
||||
# Fetch current labels on the PR
|
||||
local current
|
||||
current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name')
|
||||
|
||||
# Remove any existing label with the same prefix
|
||||
while IFS= read -r label; do
|
||||
[[ -z "$label" ]] && continue
|
||||
if [[ "$label" == "${prefix}:"* && "$label" != "$desired" ]]; then
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" 2>/dev/null || true
|
||||
fi
|
||||
done <<< "$current"
|
||||
|
||||
# Add the desired label
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "$desired"
|
||||
}
|
||||
|
||||
# ─── size ───────────────────────────────────────────────────────────────────
|
||||
|
||||
classify_size() {
|
||||
# Sum changed lines across non-doc files
|
||||
local total
|
||||
total=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \
|
||||
--paginate --jq '
|
||||
[.[] | select(.filename | test("\\.(md|txt|rst|adoc)$") | not) | .changes]
|
||||
| add // 0
|
||||
')
|
||||
|
||||
local label
|
||||
if (( total < 10 )); then label="size: XS"
|
||||
elif (( total < 50 )); then label="size: S"
|
||||
elif (( total < 200 )); then label="size: M"
|
||||
elif (( total < 500 )); then label="size: L"
|
||||
else label="size: XL"
|
||||
fi
|
||||
|
||||
echo "Size: ${total} changed lines -> ${label}"
|
||||
set_exclusive_label "size" "$label"
|
||||
}
|
||||
|
||||
# ─── risk ───────────────────────────────────────────────────────────────────
|
||||
|
||||
classify_risk() {
|
||||
# If "risk: manual" is present, skip — it's a sticky override
|
||||
local current
|
||||
current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name')
|
||||
if echo "$current" | grep -qx "risk: manual"; then
|
||||
echo "Risk: skipped (manual override)"
|
||||
return
|
||||
fi
|
||||
|
||||
# Fetch changed file paths
|
||||
local files
|
||||
files=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \
|
||||
--paginate --jq '.[].filename')
|
||||
|
||||
local risk="low"
|
||||
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
|
||||
case "$file" in
|
||||
# High risk: safety, secrets, auth, crypto, setup, orchestrator auth
|
||||
src/safety/*|src/secrets/*|src/llm/session.rs|src/orchestrator/auth.rs|\
|
||||
src/channels/web/auth.rs|src/setup/*)
|
||||
risk="high"
|
||||
break # can't go higher
|
||||
;;
|
||||
|
||||
# Medium risk: agent core, config, database, worker, tools, channels
|
||||
src/agent/*|src/config.rs|src/settings.rs|src/db/*|src/worker/*|\
|
||||
src/tools/*|src/channels/*|src/orchestrator/*|src/context/*|\
|
||||
src/hooks/*|src/sandbox/*|src/extensions/*|Cargo.toml|\
|
||||
.github/workflows/*)
|
||||
# Only upgrade, never downgrade
|
||||
[[ "$risk" != "high" ]] && risk="medium"
|
||||
;;
|
||||
|
||||
# Low risk: docs, tests, estimation, evaluation, history, etc.
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
done <<< "$files"
|
||||
|
||||
echo "Risk: ${risk}"
|
||||
set_exclusive_label "risk" "risk: ${risk}"
|
||||
}
|
||||
|
||||
# ─── contributor tier ───────────────────────────────────────────────────────
|
||||
|
||||
classify_contributor() {
|
||||
# Get PR author
|
||||
local author
|
||||
author=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json author --jq '.author.login')
|
||||
|
||||
# Count merged PRs by this author in this repo
|
||||
local count
|
||||
count=$(gh pr list --repo "$REPO" --state merged --author "$author" \
|
||||
--limit 100 --json number --jq 'length')
|
||||
|
||||
local label
|
||||
if (( count == 0 )); then label="contributor: new"
|
||||
elif (( count < 6 )); then label="contributor: regular"
|
||||
elif (( count < 20 )); then label="contributor: experienced"
|
||||
else label="contributor: core"
|
||||
fi
|
||||
|
||||
echo "Contributor: ${author} has ${count} merged PRs -> ${label}"
|
||||
set_exclusive_label "contributor" "$label"
|
||||
}
|
||||
|
||||
# ─── main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "Classifying PR #${PR_NUMBER} in ${REPO}..."
|
||||
classify_size
|
||||
classify_risk
|
||||
classify_contributor
|
||||
echo "Done."
|
||||
@@ -0,0 +1,26 @@
|
||||
name: "PR: Classify (Size, Risk, Contributor)"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: read # needed for search/issues API (contributor count)
|
||||
|
||||
jobs:
|
||||
classify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout base branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
|
||||
- name: Classify PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: bash .github/scripts/pr-labeler.sh
|
||||
@@ -0,0 +1,18 @@
|
||||
name: "PR: Scope Labels"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
scope:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@v5
|
||||
with:
|
||||
configuration-path: .github/labeler.yml
|
||||
sync-labels: false # additive only — never remove scope labels
|
||||
@@ -13,14 +13,17 @@
|
||||
### Features
|
||||
- **Multi-channel input**: TUI (Ratatui), HTTP webhooks, WASM channels (Telegram, Slack), web gateway
|
||||
- **Parallel job execution** with state machine and self-repair for stuck jobs
|
||||
- **Sandbox execution**: Docker container isolation with orchestrator/worker pattern
|
||||
- **Sandbox execution**: Docker container isolation with network proxy and credential injection
|
||||
- **Claude Code mode**: Delegate jobs to Claude CLI inside containers
|
||||
- **Skills system**: SKILL.md prompt extensions with trust model, tool attenuation, and ClawHub registry
|
||||
- **Routines**: Scheduled (cron) and reactive (event, webhook) task execution
|
||||
- **Web gateway**: Browser UI with SSE/WebSocket real-time streaming
|
||||
- **Extension management**: Install, auth, activate MCP/WASM extensions
|
||||
- **Extensible tools**: Built-in tools, WASM sandbox, MCP client, dynamic builder
|
||||
- **Persistent memory**: Workspace with hybrid search (FTS + vector via RRF)
|
||||
- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection
|
||||
- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection, shell env scrubbing
|
||||
- **Multi-provider LLM**: NEAR AI, OpenAI, Anthropic, Ollama, OpenAI-compatible, Tinfoil private inference
|
||||
- **Setup wizard**: 7-step interactive onboarding for first-run configuration
|
||||
- **Heartbeat system**: Proactive periodic execution with checklist
|
||||
|
||||
## Build & Test
|
||||
@@ -64,6 +67,7 @@ src/
|
||||
│ ├── context_monitor.rs # Memory pressure detection
|
||||
│ ├── undo.rs # Turn-based undo/redo with checkpoints
|
||||
│ ├── submission.rs # Submission parsing (undo, redo, compact, clear, etc.)
|
||||
│ ├── dispatcher.rs # Skill-aware job dispatching
|
||||
│ ├── task.rs # Sub-task execution framework
|
||||
│ ├── routine.rs # Routine types (Trigger, Action, Guardrails)
|
||||
│ └── routine_engine.rs # Routine execution (cron ticker, event matcher)
|
||||
@@ -113,11 +117,19 @@ src/
|
||||
│ ├── policy.rs # PolicyRule system with severity/actions
|
||||
│ └── leak_detector.rs # Secret detection (API keys, tokens, etc.)
|
||||
│
|
||||
├── llm/ # LLM integration (NEAR AI only)
|
||||
├── llm/ # LLM integration (multi-provider)
|
||||
│ ├── mod.rs # Provider factory, LlmBackend enum
|
||||
│ ├── provider.rs # LlmProvider trait, message types
|
||||
│ ├── nearai.rs # NEAR AI chat-api implementation
|
||||
│ ├── nearai.rs # NEAR AI Responses API provider
|
||||
│ ├── nearai_chat.rs # NEAR AI Chat Completions fallback
|
||||
│ ├── reasoning.rs # Planning, tool selection, evaluation
|
||||
│ └── session.rs # Session token management with auto-renewal
|
||||
│ ├── session.rs # Session token management with auto-renewal
|
||||
│ ├── circuit_breaker.rs # Circuit breaker for provider failures
|
||||
│ ├── retry.rs # Retry with exponential backoff
|
||||
│ ├── failover.rs # Multi-provider failover chain
|
||||
│ ├── response_cache.rs # LLM response caching
|
||||
│ ├── costs.rs # Token cost tracking
|
||||
│ └── rig_adapter.rs # Rig framework adapter
|
||||
│
|
||||
├── tools/ # Extensible tool system
|
||||
│ ├── tool.rs # Tool trait, ToolOutput, ToolError
|
||||
@@ -131,6 +143,7 @@ src/
|
||||
│ │ ├── job.rs # CreateJob, ListJobs, JobStatus, CancelJob
|
||||
│ │ ├── routine.rs # routine_create/list/update/delete/history
|
||||
│ │ ├── extension_tools.rs # Extension install/auth/activate/remove
|
||||
│ │ ├── skill_tools.rs # skill_list/search/install/remove tools
|
||||
│ │ └── marketplace.rs, ecommerce.rs, taskrabbit.rs, restaurant.rs (stubs)
|
||||
│ ├── builder/ # Dynamic tool building
|
||||
│ │ ├── core.rs # BuildRequirement, SoftwareType, Language
|
||||
@@ -180,11 +193,38 @@ src/
|
||||
│ ├── success.rs # SuccessEvaluator trait, RuleBasedEvaluator, LlmEvaluator
|
||||
│ └── metrics.rs # MetricsCollector, QualityMetrics
|
||||
│
|
||||
├── sandbox/ # Docker execution sandbox
|
||||
│ ├── mod.rs # Public API, default allowlist
|
||||
│ ├── config.rs # SandboxConfig, SandboxPolicy enum
|
||||
│ ├── manager.rs # SandboxManager orchestration
|
||||
│ ├── container.rs # ContainerRunner, Docker lifecycle
|
||||
│ ├── error.rs # SandboxError types
|
||||
│ └── proxy/ # Network proxy for containers
|
||||
│ ├── mod.rs # NetworkProxyBuilder
|
||||
│ ├── http.rs # HttpProxy, CredentialResolver trait
|
||||
│ ├── policy.rs # NetworkPolicyDecider trait
|
||||
│ └── allowlist.rs # DomainAllowlist validation
|
||||
│
|
||||
├── secrets/ # Secrets management
|
||||
│ ├── crypto.rs # AES-256-GCM encryption
|
||||
│ ├── store.rs # Secret storage
|
||||
│ └── types.rs # Credential types
|
||||
│
|
||||
├── setup/ # Onboarding wizard (spec: src/setup/README.md)
|
||||
│ ├── mod.rs # Entry point, check_onboard_needed()
|
||||
│ ├── wizard.rs # 7-step interactive wizard
|
||||
│ ├── channels.rs # Channel setup helpers
|
||||
│ └── prompts.rs # Terminal prompts (select, confirm, secret)
|
||||
│
|
||||
├── skills/ # SKILL.md prompt extension system
|
||||
│ ├── mod.rs # Core types (SkillTrust, LoadedSkill)
|
||||
│ ├── registry.rs # SkillRegistry: discover, install, remove
|
||||
│ ├── selector.rs # Deterministic scoring prefilter
|
||||
│ ├── attenuation.rs # Trust-based tool ceiling
|
||||
│ ├── gating.rs # Requirement checks (bins, env, config)
|
||||
│ ├── parser.rs # SKILL.md frontmatter + markdown parser
|
||||
│ └── catalog.rs # ClawHub registry client
|
||||
│
|
||||
└── history/ # Persistence
|
||||
├── store.rs # PostgreSQL repositories
|
||||
└── analytics.rs # Aggregation queries (JobStats, ToolStats)
|
||||
@@ -214,6 +254,7 @@ When designing new features or systems, always prefer generic/extensible archite
|
||||
- `LlmProvider` - Add new LLM backends
|
||||
- `SuccessEvaluator` - Custom evaluation logic
|
||||
- `EmbeddingProvider` - Add embedding backends (workspace search)
|
||||
- `NetworkPolicyDecider` - Custom network access policies for sandbox containers
|
||||
|
||||
### Tool Implementation
|
||||
```rust
|
||||
@@ -252,6 +293,40 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted
|
||||
\-> Failed
|
||||
```
|
||||
|
||||
### Code Style
|
||||
|
||||
- Use `crate::` imports, not `super::`
|
||||
- No `pub use` re-exports unless exposing to downstream consumers
|
||||
- Prefer strong types over strings (enums, newtypes)
|
||||
- Keep functions focused, extract helpers when logic is reused
|
||||
- Comments for non-obvious logic only
|
||||
|
||||
### Review & Fix Discipline
|
||||
|
||||
Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback.
|
||||
|
||||
**Fix the pattern, not just the instance:** When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix.
|
||||
|
||||
**Propagate architectural fixes to satellite types:** If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase.
|
||||
|
||||
**Schema translation is more than DDL:** When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for:
|
||||
- **Indexes** -- diff `CREATE INDEX` statements between the two schemas
|
||||
- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`)
|
||||
- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`)
|
||||
|
||||
**Feature flag testing:** When adding feature-gated code, test compilation with each feature in isolation:
|
||||
```bash
|
||||
cargo check # default features
|
||||
cargo check --no-default-features --features libsql # libsql only
|
||||
cargo check --all-features # all features
|
||||
```
|
||||
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||
|
||||
**Mechanical verification before committing:** Run these checks on changed files before committing:
|
||||
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables (see `.env.example`):
|
||||
@@ -263,10 +338,15 @@ LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
|
||||
# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional)
|
||||
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
|
||||
|
||||
# NEAR AI (required)
|
||||
NEARAI_SESSION_TOKEN=sess_...
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
# NEAR AI (when LLM_BACKEND=nearai, the default)
|
||||
# Two modes: "NEAR AI Chat" (session token) or "NEAR AI Cloud" (API key)
|
||||
# NEAR AI Chat (Responses API, default):
|
||||
NEARAI_SESSION_TOKEN=sess_... # session token for chat-api
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
# NEAR AI Cloud (Chat Completions API, auto-selected when API key is set):
|
||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||
# NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
# Agent settings
|
||||
AGENT_NAME=ironclaw
|
||||
@@ -297,6 +377,10 @@ SANDBOX_ENABLED=true
|
||||
SANDBOX_IMAGE=ironclaw-worker:latest
|
||||
SANDBOX_MEMORY_LIMIT_MB=512
|
||||
SANDBOX_TIMEOUT_SECS=1800
|
||||
SANDBOX_CPU_LIMIT=1.0 # CPU cores per container
|
||||
SANDBOX_NETWORK_PROXY=true # Enable network proxy for containers
|
||||
SANDBOX_PROXY_PORT=8080 # Proxy listener port
|
||||
SANDBOX_DEFAULT_POLICY=workspace_write # ReadOnly, WorkspaceWrite, FullAccess
|
||||
|
||||
# Claude Code mode (runs inside sandbox containers)
|
||||
CLAUDE_CODE_ENABLED=false
|
||||
@@ -308,16 +392,27 @@ CLAUDE_CODE_CONFIG_DIR=/home/worker/.claude
|
||||
ROUTINES_ENABLED=true
|
||||
ROUTINES_CRON_INTERVAL=60 # Tick interval in seconds
|
||||
ROUTINES_MAX_CONCURRENT=3
|
||||
|
||||
# Skills system
|
||||
SKILLS_ENABLED=true
|
||||
SKILLS_MAX_TOKENS=4000 # Max prompt budget per turn
|
||||
SKILLS_CATALOG_URL=https://clawhub.dev # ClawHub registry URL
|
||||
SKILLS_AUTO_DISCOVER=true # Scan skill directories on startup
|
||||
|
||||
# Tinfoil private inference
|
||||
TINFOIL_API_KEY=... # Required when LLM_BACKEND=tinfoil
|
||||
TINFOIL_MODEL=kimi-k2-5 # Default model
|
||||
```
|
||||
|
||||
### NEAR AI Provider
|
||||
### LLM Providers
|
||||
|
||||
Uses the NEAR AI chat-api (`https://api.near.ai/v1/responses`) which provides:
|
||||
- Unified access to multiple models (OpenAI, Anthropic, etc.)
|
||||
- User authentication via session tokens
|
||||
- Usage tracking and billing through NEAR AI
|
||||
IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, and `tinfoil`.
|
||||
|
||||
Session tokens have the format `sess_xxx` (37 characters). They are authenticated against the NEAR AI auth service.
|
||||
**NEAR AI Chat** -- Uses the NEAR AI Responses API (`https://private.near.ai/v1/responses`). Authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google). Supports response chaining (delta-only follow-up messages) for efficient multi-turn conversations. This is the default mode when no `NEARAI_API_KEY` is set. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment. Configure with `NEARAI_BASE_URL` (default: `https://private.near.ai`).
|
||||
|
||||
**NEAR AI Cloud** -- Uses the OpenAI-compatible Chat Completions API (`https://cloud-api.near.ai/v1/chat/completions`). Authenticates with API keys from `cloud.near.ai`. Auto-selected when `NEARAI_API_KEY` is set (or explicitly via `NEARAI_API_MODE=chat_completions`). Tool messages are flattened to plain text for compatibility. Configure with `NEARAI_API_KEY` and `NEARAI_BASE_URL` (default: `https://cloud-api.near.ai`).
|
||||
|
||||
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API only (not the Responses API, so tool calls are adapted to chat format). Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
|
||||
|
||||
## Database
|
||||
|
||||
@@ -386,22 +481,7 @@ Both backends implement this trait. PostgreSQL delegates to the existing `Store`
|
||||
- `tool_failures` - Self-repair tracking
|
||||
- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure
|
||||
|
||||
### Configuration
|
||||
|
||||
```bash
|
||||
# Backend selection (default: postgres)
|
||||
DATABASE_BACKEND=libsql
|
||||
|
||||
# PostgreSQL
|
||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||
|
||||
# libSQL (embedded)
|
||||
LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path
|
||||
|
||||
# libSQL (Turso cloud sync)
|
||||
LIBSQL_URL=libsql://your-db.turso.io
|
||||
LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set
|
||||
```
|
||||
Database configuration: see Configuration section above.
|
||||
|
||||
### Current Limitations (libSQL backend)
|
||||
|
||||
@@ -419,6 +499,7 @@ All external tool output passes through `SafetyLayer`:
|
||||
1. **Sanitizer** - Detects injection patterns, escapes dangerous content
|
||||
2. **Validator** - Checks length, encoding, forbidden patterns
|
||||
3. **Policy** - Rules with severity (Critical/High/Medium/Low) and actions (Block/Warn/Review/Sanitize)
|
||||
4. **Leak Detector** - Scans for 15+ secret patterns (API keys, tokens, private keys, connection strings) at two points: tool output before it reaches the LLM, and LLM responses before they reach the user. Actions per pattern: Block (reject entirely), Redact (mask the secret), or Warn (flag but allow)
|
||||
|
||||
Tool outputs are wrapped before reaching LLM:
|
||||
```xml
|
||||
@@ -427,6 +508,95 @@ Tool outputs are wrapped before reaching LLM:
|
||||
</tool_output>
|
||||
```
|
||||
|
||||
### Shell Environment Scrubbing
|
||||
|
||||
The shell tool (`src/tools/builtin/shell.rs`) scrubs sensitive environment variables before executing commands, preventing secrets from leaking through `env`, `printenv`, or `$VAR` expansion. The sanitizer (`src/safety/sanitizer.rs`) also detects command injection patterns (chained commands, subshells, path traversal) and blocks or escapes them based on policy rules.
|
||||
|
||||
## Skills System
|
||||
|
||||
Skills are SKILL.md files that extend the agent's prompt with domain-specific instructions. Each skill is a YAML frontmatter block (metadata, activation criteria, required tools) followed by a markdown body that gets injected into the LLM context when the skill activates.
|
||||
|
||||
### Trust Model
|
||||
|
||||
| Trust Level | Source | Tool Access |
|
||||
|-------------|--------|-------------|
|
||||
| **Trusted** | User-placed in `~/.ironclaw/skills/` or workspace `skills/` | All tools available to the agent |
|
||||
| **Installed** | Downloaded from ClawHub registry | Read-only tools only (no shell, file write, HTTP) |
|
||||
|
||||
### SKILL.md Format
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: my-skill
|
||||
version: 0.1.0
|
||||
description: Does something useful
|
||||
activation:
|
||||
patterns:
|
||||
- "deploy to.*production"
|
||||
keywords:
|
||||
- "deployment"
|
||||
max_context_tokens: 2000
|
||||
metadata:
|
||||
openclaw:
|
||||
requires:
|
||||
bins: [docker, kubectl]
|
||||
env: [KUBECONFIG]
|
||||
---
|
||||
|
||||
# Deployment Skill
|
||||
|
||||
Instructions for the agent when this skill activates...
|
||||
```
|
||||
|
||||
### Selection Pipeline
|
||||
|
||||
1. **Gating** -- Check binary/env/config requirements; skip skills whose prerequisites are missing
|
||||
2. **Scoring** -- Deterministic scoring against message content using keywords, tags, and regex patterns
|
||||
3. **Budget** -- Select top-scoring skills that fit within `SKILLS_MAX_TOKENS` prompt budget
|
||||
4. **Attenuation** -- Apply trust-based tool ceiling; installed skills lose access to dangerous tools
|
||||
|
||||
### Skill Tools
|
||||
|
||||
Four built-in tools for managing skills at runtime:
|
||||
- **`skill_list`** -- List all discovered skills with trust level and status
|
||||
- **`skill_search`** -- Search ClawHub registry for available skills
|
||||
- **`skill_install`** -- Download and install a skill from ClawHub
|
||||
- **`skill_remove`** -- Remove an installed skill
|
||||
|
||||
### Skill Directories
|
||||
|
||||
- `~/.ironclaw/skills/` -- User's global skills (trusted)
|
||||
- `<workspace>/skills/` -- Per-workspace skills (trusted)
|
||||
- `~/.ironclaw/installed_skills/` -- Registry-installed skills (installed trust)
|
||||
|
||||
Skills configuration: see Configuration section above.
|
||||
|
||||
## Docker Sandbox
|
||||
|
||||
The `src/sandbox/` module provides Docker-based isolation for job execution with a network proxy that controls outbound access and injects credentials.
|
||||
|
||||
### Sandbox Policies
|
||||
|
||||
| Policy | Filesystem | Network | Use Case |
|
||||
|--------|-----------|---------|----------|
|
||||
| **ReadOnly** | Read-only workspace mount | Allowlisted domains only | Analysis, code review |
|
||||
| **WorkspaceWrite** | Read-write workspace mount | Allowlisted domains only | Code generation, file edits |
|
||||
| **FullAccess** | Full filesystem | Unrestricted | Trusted admin tasks |
|
||||
|
||||
### Network Proxy
|
||||
|
||||
Containers route all HTTP/HTTPS traffic through a host-side proxy (`src/sandbox/proxy/`):
|
||||
- **Domain allowlist** -- Only allowlisted domains are reachable (default: package registries, docs sites, GitHub, common APIs)
|
||||
- **Credential injection** -- The `CredentialResolver` trait injects auth headers into proxied requests so secrets never enter the container environment
|
||||
- **CONNECT tunnel** -- HTTPS traffic uses CONNECT method; the proxy validates the target domain against the allowlist before establishing the tunnel
|
||||
- **Policy decisions** -- The `NetworkPolicyDecider` trait allows custom logic for allow/deny/inject decisions per request
|
||||
|
||||
### Zero-Exposure Credential Model
|
||||
|
||||
Secrets (API keys, tokens) are stored encrypted on the host and injected into HTTP requests by the proxy at transit time. Container processes never have access to raw credential values, preventing exfiltration even if container code is compromised.
|
||||
|
||||
Sandbox configuration: see Configuration section above.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests are in `mod tests {}` blocks at the bottom of each file. Run specific module tests:
|
||||
@@ -451,164 +621,13 @@ Key test patterns:
|
||||
7. **Webhook trigger endpoint** - Routines webhook trigger not yet exposed in web gateway
|
||||
8. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard
|
||||
|
||||
### Completed
|
||||
## Tool Architecture
|
||||
|
||||
- ✅ **Workspace integration** - Memory tools registered, workspace passed to Agent and heartbeat
|
||||
- ✅ **WASM sandboxing** - Full implementation in `tools/wasm/` with fuel metering, memory limits, capabilities
|
||||
- ✅ **Dynamic tool building** - `tools/builder/` has LlmSoftwareBuilder with iterative build loop
|
||||
- ✅ **HTTP webhook security** - Secret validation implemented, proper error handling (no panics)
|
||||
- ✅ **Embeddings integration** - OpenAI and NEAR AI providers wired to workspace for semantic search
|
||||
- ✅ **Workspace system prompt** - Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) injected into LLM context
|
||||
- ✅ **Heartbeat notifications** - Route through channel manager (broadcast API) instead of logging-only
|
||||
- ✅ **Auto-context compaction** - Triggers automatically when context exceeds threshold
|
||||
- ✅ **Embedding backfill** - Runs on startup when embeddings provider is enabled
|
||||
- ✅ **Clippy clean** - All warnings addressed via config struct refactoring
|
||||
- ✅ **Tool approval enforcement** - Tools with `requires_approval()` (shell, http, file write/patch, build_software) now gate execution, track auto-approved tools per session
|
||||
- ✅ **Tool definition refresh** - Tool definitions refreshed each iteration so newly built tools become visible in same session
|
||||
- ✅ **Worker tool call handling** - Uses `respond_with_tools()` to properly execute tool calls when `select_tools()` returns empty
|
||||
- ✅ **Gateway control plane** - Web gateway with 40+ API endpoints, SSE/WebSocket
|
||||
- ✅ **Web Control UI** - Browser-based dashboard with chat, memory, jobs, logs, extensions, routines
|
||||
- ✅ **Slack/Telegram channels** - Implemented as WASM tools
|
||||
- ✅ **Docker sandbox** - Orchestrator/worker containers with per-job auth
|
||||
- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers
|
||||
- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails
|
||||
- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI
|
||||
- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode
|
||||
**Keep tool-specific logic out of the main agent codebase.** The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through `capabilities.json` files (API endpoints, credentials, rate limits, auth setup). Service-specific auth flows, CLI commands, and configuration do not belong in the main agent.
|
||||
|
||||
## Adding a New Tool
|
||||
Tools can be built as **WASM** (sandboxed, credential-injected, single binary) or **MCP servers** (ecosystem of pre-built servers, any language, but no sandbox). Both are first-class via `ironclaw tool install`. Auth is declared in capabilities files with OAuth and manual token entry support.
|
||||
|
||||
### Built-in Tools (Rust)
|
||||
|
||||
1. Create `src/tools/builtin/my_tool.rs`
|
||||
2. Implement the `Tool` trait
|
||||
3. Add `mod my_tool;` and `pub use` in `src/tools/builtin/mod.rs`
|
||||
4. Register in `ToolRegistry::register_builtin_tools()` in `registry.rs`
|
||||
5. Add tests
|
||||
|
||||
### WASM Tools (Recommended)
|
||||
|
||||
WASM tools are the preferred way to add new capabilities. They run in a sandboxed environment with explicit capabilities.
|
||||
|
||||
1. Create a new crate in `tools-src/<name>/`
|
||||
2. Implement the WIT interface (`wit/tool.wit`)
|
||||
3. Create `<name>.capabilities.json` declaring required permissions
|
||||
4. Build with `cargo build --target wasm32-wasip2 --release`
|
||||
5. Install with `ironclaw tool install path/to/tool.wasm`
|
||||
|
||||
See `tools-src/` for examples.
|
||||
|
||||
## Tool Architecture Principles
|
||||
|
||||
**CRITICAL: Keep tool-specific logic out of the main agent codebase.**
|
||||
|
||||
The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through capabilities files.
|
||||
|
||||
### What Goes in Tools (capabilities.json)
|
||||
|
||||
- API endpoints the tool needs (HTTP allowlist)
|
||||
- Credentials required (secret names, injection locations)
|
||||
- Rate limits and timeouts
|
||||
- Auth setup instructions (see below)
|
||||
- Workspace paths the tool can read
|
||||
|
||||
### What Does NOT Go in Main Agent
|
||||
|
||||
- Service-specific auth flows (OAuth for Notion, Slack, etc.)
|
||||
- Service-specific CLI commands (`auth notion`, `auth slack`)
|
||||
- Service-specific configuration handling
|
||||
- Hardcoded API URLs or token formats
|
||||
|
||||
### Tool Authentication
|
||||
|
||||
Tools declare their auth requirements in `<tool>.capabilities.json` under the `auth` section. Two methods are supported:
|
||||
|
||||
#### OAuth (Browser-based login)
|
||||
|
||||
For services that support OAuth, users just click through browser login:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "notion_api_token",
|
||||
"display_name": "Notion",
|
||||
"oauth": {
|
||||
"authorization_url": "https://api.notion.com/v1/oauth/authorize",
|
||||
"token_url": "https://api.notion.com/v1/oauth/token",
|
||||
"client_id_env": "NOTION_OAUTH_CLIENT_ID",
|
||||
"client_secret_env": "NOTION_OAUTH_CLIENT_SECRET",
|
||||
"scopes": [],
|
||||
"use_pkce": false,
|
||||
"extra_params": { "owner": "user" }
|
||||
},
|
||||
"env_var": "NOTION_TOKEN"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To enable OAuth for a tool:
|
||||
1. Register a public OAuth app with the service (e.g., notion.so/my-integrations)
|
||||
2. Configure redirect URIs: `http://localhost:9876/callback` through `http://localhost:9886/callback`
|
||||
3. Set environment variables for client_id and client_secret
|
||||
|
||||
#### Manual Token Entry (Fallback)
|
||||
|
||||
For services without OAuth or when OAuth isn't configured:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "openai_api_key",
|
||||
"display_name": "OpenAI",
|
||||
"instructions": "Get your API key from platform.openai.com/api-keys",
|
||||
"setup_url": "https://platform.openai.com/api-keys",
|
||||
"token_hint": "Starts with 'sk-'",
|
||||
"env_var": "OPENAI_API_KEY"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Auth Flow Priority
|
||||
|
||||
When running `ironclaw tool auth <tool>`:
|
||||
|
||||
1. Check `env_var` - if set in environment, use it directly
|
||||
2. Check `oauth` - if configured, open browser for OAuth flow
|
||||
3. Fall back to `instructions` + manual token entry
|
||||
|
||||
The agent reads auth config from the tool's capabilities file and provides the appropriate flow. No service-specific code in the main agent.
|
||||
|
||||
### WASM Tools vs MCP Servers: When to Use Which
|
||||
|
||||
Both are first-class in the extension system (`ironclaw tool install` handles both), but they have different strengths.
|
||||
|
||||
**WASM Tools (IronClaw native)**
|
||||
|
||||
- Sandboxed: fuel metering, memory limits, no access except what's allowlisted
|
||||
- Credentials injected by host runtime, tool code never sees the actual token
|
||||
- Output scanned for secret leakage before returning to the LLM
|
||||
- Auth (OAuth/manual) declared in `capabilities.json`, agent handles the flow
|
||||
- Single binary, no process management, works offline
|
||||
- Cost: must build yourself in Rust, no ecosystem, synchronous only
|
||||
|
||||
**MCP Servers (Model Context Protocol)**
|
||||
|
||||
- Growing ecosystem of pre-built servers (GitHub, Notion, Postgres, etc.)
|
||||
- Any language (TypeScript/Python most common)
|
||||
- Can do websockets, streaming, background polling
|
||||
- Cost: external process with full system access (no sandbox), manages own credentials, IronClaw can't prevent leaks
|
||||
|
||||
**Decision guide:**
|
||||
|
||||
| Scenario | Use |
|
||||
|----------|-----|
|
||||
| Good MCP server already exists | **MCP** |
|
||||
| Handles sensitive credentials (email send, banking) | **WASM** |
|
||||
| Quick prototype or one-off integration | **MCP** |
|
||||
| Core capability you'll maintain long-term | **WASM** |
|
||||
| Needs background connections (websockets, polling) | **MCP** |
|
||||
| Multiple tools share one OAuth token (e.g., Google suite) | **WASM** |
|
||||
|
||||
The LLM-facing interface is identical for both (tool name, schema, execute), so swapping between them is transparent to the agent.
|
||||
See `src/tools/README.md` for full tool architecture, adding new tools (built-in Rust and WASM), auth JSON examples, and WASM vs MCP decision guide.
|
||||
|
||||
## Adding a New Channel
|
||||
|
||||
@@ -645,154 +664,15 @@ for that module's behavior. When modifying code in a module that has a spec:
|
||||
| Module | Spec File |
|
||||
|--------|-----------|
|
||||
| `src/setup/` | `src/setup/README.md` |
|
||||
|
||||
## Code Style
|
||||
|
||||
- Use `crate::` imports, not `super::`
|
||||
- No `pub use` re-exports unless exposing to downstream consumers
|
||||
- Prefer strong types over strings (enums, newtypes)
|
||||
- Keep functions focused, extract helpers when logic is reused
|
||||
- Comments for non-obvious logic only
|
||||
|
||||
## Review & Fix Discipline
|
||||
|
||||
Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback.
|
||||
|
||||
### Fix the pattern, not just the instance
|
||||
When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix.
|
||||
|
||||
### Propagate architectural fixes to satellite types
|
||||
If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase.
|
||||
|
||||
### Schema translation is more than DDL
|
||||
When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for:
|
||||
- **Indexes** -- diff `CREATE INDEX` statements between the two schemas
|
||||
- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`)
|
||||
- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`)
|
||||
|
||||
### Feature flag testing
|
||||
When adding feature-gated code, test compilation with each feature in isolation:
|
||||
```bash
|
||||
cargo check # default features
|
||||
cargo check --no-default-features --features libsql # libsql only
|
||||
cargo check --all-features # all features
|
||||
```
|
||||
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||
|
||||
### Mechanical verification before committing
|
||||
Run these checks on changed files before committing:
|
||||
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||
| `src/workspace/` | `src/workspace/README.md` |
|
||||
| `src/tools/` | `src/tools/README.md` |
|
||||
|
||||
## Workspace & Memory System
|
||||
|
||||
Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure.
|
||||
OpenClaw-inspired persistent memory with a flexible filesystem-like structure. Principle: "Memory is database, not RAM" -- if you want to remember something, write it explicitly. Uses hybrid search combining FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion.
|
||||
|
||||
### Key Principles
|
||||
Four memory tools for LLM use: `memory_search` (hybrid search -- call before answering questions about prior work), `memory_write`, `memory_read`, `memory_tree`. Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) are injected into the LLM system prompt.
|
||||
|
||||
1. **"Memory is database, not RAM"** - If you want to remember something, write it explicitly
|
||||
2. **Flexible structure** - Create any directory/file hierarchy you need
|
||||
3. **Self-documenting** - Use README.md files to describe directory structure
|
||||
4. **Hybrid search** - Combines FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion
|
||||
The heartbeat system runs proactive periodic execution (default: 30 minutes), reading `HEARTBEAT.md` and notifying via channel if findings are detected.
|
||||
|
||||
### Filesystem Structure
|
||||
|
||||
```
|
||||
workspace/
|
||||
├── README.md <- Root runbook/index
|
||||
├── MEMORY.md <- Long-term curated memory
|
||||
├── HEARTBEAT.md <- Periodic checklist
|
||||
├── IDENTITY.md <- Agent name, nature, vibe
|
||||
├── SOUL.md <- Core values
|
||||
├── AGENTS.md <- Behavior instructions
|
||||
├── USER.md <- User context
|
||||
├── context/ <- Identity-related docs
|
||||
│ ├── vision.md
|
||||
│ └── priorities.md
|
||||
├── daily/ <- Daily logs
|
||||
│ ├── 2024-01-15.md
|
||||
│ └── 2024-01-16.md
|
||||
├── projects/ <- Arbitrary structure
|
||||
│ └── alpha/
|
||||
│ ├── README.md
|
||||
│ └── notes.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
### Using the Workspace
|
||||
|
||||
```rust
|
||||
use crate::workspace::{Workspace, OpenAiEmbeddings, paths};
|
||||
|
||||
// Create workspace for a user
|
||||
let workspace = Workspace::new("user_123", pool)
|
||||
.with_embeddings(Arc::new(OpenAiEmbeddings::new(api_key)));
|
||||
|
||||
// Read/write any path
|
||||
let doc = workspace.read("projects/alpha/notes.md").await?;
|
||||
workspace.write("context/priorities.md", "# Priorities\n\n1. Feature X").await?;
|
||||
workspace.append("daily/2024-01-15.md", "Completed task X").await?;
|
||||
|
||||
// Convenience methods for well-known files
|
||||
workspace.append_memory("User prefers dark mode").await?;
|
||||
workspace.append_daily_log("Session note").await?;
|
||||
|
||||
// List directory contents
|
||||
let entries = workspace.list("projects/").await?;
|
||||
|
||||
// Search (hybrid FTS + vector)
|
||||
let results = workspace.search("dark mode preference", 5).await?;
|
||||
|
||||
// Get system prompt from identity files
|
||||
let prompt = workspace.system_prompt().await?;
|
||||
```
|
||||
|
||||
### Memory Tools
|
||||
|
||||
Four tools for LLM use:
|
||||
|
||||
- **`memory_search`** - Hybrid search, MUST be called before answering questions about prior work
|
||||
- **`memory_write`** - Write to any path (memory, daily_log, or custom paths)
|
||||
- **`memory_read`** - Read any file by path
|
||||
- **`memory_tree`** - View workspace structure as a tree (depth parameter, default 1)
|
||||
|
||||
### Hybrid Search (RRF)
|
||||
|
||||
Combines full-text search and vector similarity using Reciprocal Rank Fusion:
|
||||
|
||||
```
|
||||
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||
```
|
||||
|
||||
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
||||
|
||||
**Backend differences:**
|
||||
- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF
|
||||
- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired)
|
||||
|
||||
### Heartbeat System
|
||||
|
||||
Proactive periodic execution (default: 30 minutes):
|
||||
|
||||
1. Reads `HEARTBEAT.md` checklist
|
||||
2. Runs agent turn with checklist prompt
|
||||
3. If findings, notifies via channel
|
||||
4. If nothing, agent replies "HEARTBEAT_OK" (no notification)
|
||||
|
||||
```rust
|
||||
use crate::agent::{HeartbeatConfig, spawn_heartbeat};
|
||||
|
||||
let config = HeartbeatConfig::default()
|
||||
.with_interval(Duration::from_secs(60 * 30))
|
||||
.with_notify("user_123", "telegram");
|
||||
|
||||
spawn_heartbeat(config, workspace, llm, response_tx);
|
||||
```
|
||||
|
||||
### Chunking Strategy
|
||||
|
||||
Documents are chunked for search indexing:
|
||||
- Default: 800 words per chunk (roughly 800 tokens for English)
|
||||
- 15% overlap between chunks for context preservation
|
||||
- Minimum chunk size: 50 words (tiny trailing chunks merge with previous)
|
||||
See `src/workspace/README.md` for full API documentation, filesystem structure, hybrid search details, chunking strategy, and heartbeat system.
|
||||
|
||||
+10
@@ -1,10 +1,20 @@
|
||||
[workspace]
|
||||
members = [".", "benchmarks"]
|
||||
exclude = [
|
||||
"channels-src/discord",
|
||||
"channels-src/telegram",
|
||||
"channels-src/slack",
|
||||
"channels-src/whatsapp",
|
||||
"tools-src/github",
|
||||
"tools-src/gmail",
|
||||
"tools-src/google-calendar",
|
||||
"tools-src/google-docs",
|
||||
"tools-src/google-drive",
|
||||
"tools-src/google-sheets",
|
||||
"tools-src/google-slides",
|
||||
"tools-src/okta",
|
||||
"tools-src/slack",
|
||||
"tools-src/telegram",
|
||||
]
|
||||
|
||||
[package]
|
||||
|
||||
@@ -402,7 +402,7 @@ async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult {
|
||||
let mut channels = ChannelManager::new();
|
||||
channels.add(Box::new(bench_channel));
|
||||
|
||||
let agent = Agent::new(agent_config, deps, channels, None, None, None, None);
|
||||
let agent = Agent::new(agent_config, deps, channels, None, None, None, None, None);
|
||||
|
||||
// Build the full prompt with context
|
||||
let full_prompt = if let Some(ref ctx) = task.context {
|
||||
|
||||
@@ -21,3 +21,5 @@ lto = true
|
||||
codegen-units = 1
|
||||
|
||||
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -27,3 +27,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -25,3 +25,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -16,3 +16,5 @@ serde_json = "1"
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
|
||||
[workspace]
|
||||
|
||||
+8
-5
@@ -2,12 +2,15 @@
|
||||
# Do not use placeholder passwords in production.
|
||||
DATABASE_URL=postgres://ironclaw:CHANGE_ME@localhost:5432/ironclaw
|
||||
|
||||
# NEAR AI
|
||||
NEARAI_SESSION_TOKEN=CHANGE_ME
|
||||
# NEAR AI Cloud (API key auth, Chat Completions API)
|
||||
# Get an API key from https://cloud.near.ai
|
||||
NEARAI_API_KEY=CHANGE_ME
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
NEARAI_AUTH_URL=https://private.near.ai
|
||||
NEARAI_API_MODE=chat_completions
|
||||
NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||
|
||||
# Or use NEAR AI Chat (session token auth, Responses API):
|
||||
# NEARAI_SESSION_TOKEN=sess_...
|
||||
# NEARAI_BASE_URL=https://private.near.ai
|
||||
|
||||
# Agent
|
||||
AGENT_NAME=ironclaw
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"bundles": {
|
||||
"google": {
|
||||
"display_name": "Google Suite",
|
||||
"description": "Gmail, Calendar, Drive, Docs, Sheets, Slides",
|
||||
"extensions": [
|
||||
"tools/gmail",
|
||||
"tools/google-calendar",
|
||||
"tools/google-docs",
|
||||
"tools/google-drive",
|
||||
"tools/google-sheets",
|
||||
"tools/google-slides"
|
||||
],
|
||||
"shared_auth": "google_oauth_token"
|
||||
},
|
||||
"messaging": {
|
||||
"display_name": "Messaging Channels",
|
||||
"description": "Discord, Telegram, Slack, and WhatsApp channels",
|
||||
"extensions": [
|
||||
"channels/discord",
|
||||
"channels/telegram",
|
||||
"channels/slack",
|
||||
"channels/whatsapp"
|
||||
],
|
||||
"shared_auth": null
|
||||
},
|
||||
"default": {
|
||||
"display_name": "Recommended Set",
|
||||
"description": "Core tools and channels for a productive setup",
|
||||
"extensions": [
|
||||
"tools/github",
|
||||
"tools/gmail",
|
||||
"tools/google-calendar",
|
||||
"tools/google-drive",
|
||||
"tools/slack",
|
||||
"channels/telegram",
|
||||
"channels/slack"
|
||||
],
|
||||
"shared_auth": null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "discord",
|
||||
"display_name": "Discord",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Discord Gateway/Webhook channel for slash commands, buttons, and messages",
|
||||
"keywords": ["messaging", "chat", "discord", "bot"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/discord",
|
||||
"capabilities": "discord.capabilities.json",
|
||||
"crate_name": "discord-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Discord",
|
||||
"secrets": ["discord_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://discord.com/developers/applications"
|
||||
},
|
||||
|
||||
"tags": ["messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Slack Events API channel for receiving and responding to Slack messages",
|
||||
"keywords": ["messaging", "chat", "workspace", "slack"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/slack",
|
||||
"capabilities": "slack.capabilities.json",
|
||||
"crate_name": "slack-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token", "slack_signing_secret"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://api.slack.com/apps"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "telegram",
|
||||
"display_name": "Telegram",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Telegram Bot API channel for receiving and responding to messages",
|
||||
"keywords": ["messaging", "bot", "chat", "telegram"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/telegram",
|
||||
"capabilities": "telegram.capabilities.json",
|
||||
"crate_name": "telegram-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Telegram",
|
||||
"secrets": ["telegram_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://t.me/BotFather"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "whatsapp",
|
||||
"display_name": "WhatsApp",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "WhatsApp Cloud API channel for receiving and responding to messages",
|
||||
"keywords": ["messaging", "chat", "whatsapp", "meta"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/whatsapp",
|
||||
"capabilities": "whatsapp.capabilities.json",
|
||||
"crate_name": "whatsapp-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Meta",
|
||||
"secrets": ["whatsapp_access_token", "whatsapp_verify_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://developers.facebook.com/apps/"
|
||||
},
|
||||
|
||||
"tags": ["messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "github",
|
||||
"display_name": "GitHub",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "GitHub integration for issues, PRs, repos, and code search",
|
||||
"keywords": ["git", "code", "issues", "pull-requests", "repositories"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/github",
|
||||
"capabilities": "github-tool.capabilities.json",
|
||||
"crate_name": "github-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "GitHub",
|
||||
"secrets": ["github_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://github.com/settings/tokens"
|
||||
},
|
||||
|
||||
"tags": ["default", "development"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "gmail",
|
||||
"display_name": "Gmail",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Read, send, and manage Gmail messages and threads",
|
||||
"keywords": ["email", "google", "mail", "messaging"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/gmail",
|
||||
"capabilities": "gmail-tool.capabilities.json",
|
||||
"crate_name": "gmail-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-calendar",
|
||||
"display_name": "Google Calendar",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create, read, update, and delete Google Calendar events",
|
||||
"keywords": ["calendar", "google", "scheduling", "events"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-calendar",
|
||||
"capabilities": "google-calendar-tool.capabilities.json",
|
||||
"crate_name": "google-calendar-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-docs",
|
||||
"display_name": "Google Docs",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create and edit Google Docs documents",
|
||||
"keywords": ["documents", "google", "writing", "docs"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-docs",
|
||||
"capabilities": "google-docs-tool.capabilities.json",
|
||||
"crate_name": "google-docs-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-drive",
|
||||
"display_name": "Google Drive",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Upload, download, search, and manage Google Drive files and folders",
|
||||
"keywords": ["storage", "google", "files", "drive"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-drive",
|
||||
"capabilities": "google-drive-tool.capabilities.json",
|
||||
"crate_name": "google-drive-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "storage"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-sheets",
|
||||
"display_name": "Google Sheets",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Read and write Google Sheets spreadsheet data",
|
||||
"keywords": ["spreadsheets", "google", "data", "sheets"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-sheets",
|
||||
"capabilities": "google-sheets-tool.capabilities.json",
|
||||
"crate_name": "google-sheets-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-slides",
|
||||
"display_name": "Google Slides",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create and edit Google Slides presentations",
|
||||
"keywords": ["presentations", "google", "slides"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-slides",
|
||||
"capabilities": "google-slides-tool.capabilities.json",
|
||||
"crate_name": "google-slides-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "okta",
|
||||
"display_name": "Okta",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Okta SSO for user profile, app catalog, and SSO launch links",
|
||||
"keywords": ["sso", "identity", "authentication", "okta"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/okta",
|
||||
"capabilities": "okta-tool.capabilities.json",
|
||||
"crate_name": "okta-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Okta",
|
||||
"secrets": ["okta_oauth_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/"
|
||||
},
|
||||
|
||||
"tags": ["identity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Post messages, read channels, and manage conversations via Slack API",
|
||||
"keywords": ["messaging", "chat", "workspace"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/slack",
|
||||
"capabilities": "slack-tool.capabilities.json",
|
||||
"crate_name": "slack-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://api.slack.com/apps"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "telegram",
|
||||
"display_name": "Telegram",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Telegram user-mode integration via MTProto for messages and contacts",
|
||||
"keywords": ["messaging", "chat", "telegram", "mtproto"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/telegram",
|
||||
"capabilities": "telegram-tool.capabilities.json",
|
||||
"crate_name": "telegram-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Telegram",
|
||||
"secrets": ["telegram_api_id", "telegram_api_hash"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://my.telegram.org/apps"
|
||||
},
|
||||
|
||||
"tags": ["messaging"]
|
||||
}
|
||||
@@ -85,6 +85,7 @@ pub struct Agent {
|
||||
pub(super) session_manager: Arc<SessionManager>,
|
||||
pub(super) context_monitor: ContextMonitor,
|
||||
pub(super) heartbeat_config: Option<HeartbeatConfig>,
|
||||
pub(super) hygiene_config: Option<crate::config::HygieneConfig>,
|
||||
pub(super) routine_config: Option<RoutineConfig>,
|
||||
}
|
||||
|
||||
@@ -93,11 +94,13 @@ impl Agent {
|
||||
///
|
||||
/// Optionally accepts pre-created `ContextManager` and `SessionManager` for sharing
|
||||
/// with external components (job tools, web gateway). Creates new ones if not provided.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
config: AgentConfig,
|
||||
deps: AgentDeps,
|
||||
channels: ChannelManager,
|
||||
heartbeat_config: Option<HeartbeatConfig>,
|
||||
hygiene_config: Option<crate::config::HygieneConfig>,
|
||||
routine_config: Option<RoutineConfig>,
|
||||
context_manager: Option<Arc<ContextManager>>,
|
||||
session_manager: Option<Arc<SessionManager>>,
|
||||
@@ -127,6 +130,7 @@ impl Agent {
|
||||
session_manager,
|
||||
context_monitor: ContextMonitor::new(),
|
||||
heartbeat_config,
|
||||
hygiene_config,
|
||||
routine_config,
|
||||
}
|
||||
}
|
||||
@@ -358,8 +362,15 @@ impl Agent {
|
||||
"Heartbeat enabled with {}s interval",
|
||||
hb_config.interval_secs
|
||||
);
|
||||
let hygiene = self
|
||||
.hygiene_config
|
||||
.as_ref()
|
||||
.map(|h| h.to_workspace_config())
|
||||
.unwrap_or_default();
|
||||
|
||||
Some(spawn_heartbeat(
|
||||
config,
|
||||
hygiene,
|
||||
workspace.clone(),
|
||||
self.cheap_llm().clone(),
|
||||
Some(notify_tx),
|
||||
|
||||
@@ -232,6 +232,7 @@ impl Agent {
|
||||
|
||||
let runner = crate::agent::HeartbeatRunner::new(
|
||||
crate::agent::HeartbeatConfig::default(),
|
||||
crate::workspace::hygiene::HygieneConfig::default(),
|
||||
workspace.clone(),
|
||||
self.llm().clone(),
|
||||
);
|
||||
|
||||
+20
-2
@@ -105,7 +105,16 @@ impl ContextCompactor {
|
||||
|
||||
// Write to workspace if available
|
||||
let summary_written = if let Some(ws) = workspace {
|
||||
self.write_summary_to_workspace(ws, &summary).await.is_ok()
|
||||
match self.write_summary_to_workspace(ws, &summary).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Compaction summary write failed (turns will still be truncated): {}",
|
||||
e
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
};
|
||||
@@ -157,7 +166,16 @@ impl ContextCompactor {
|
||||
let content = format_turns_for_storage(old_turns);
|
||||
|
||||
// Write to workspace
|
||||
let written = self.write_context_to_workspace(ws, &content).await.is_ok();
|
||||
let written = match self.write_context_to_workspace(ws, &content).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Compaction context write failed (turns will still be truncated): {}",
|
||||
e
|
||||
);
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
// Truncate
|
||||
thread.truncate_turns(keep_recent);
|
||||
|
||||
+686
-250
File diff suppressed because it is too large
Load Diff
+22
-1
@@ -31,6 +31,7 @@ use tokio::sync::mpsc;
|
||||
use crate::channels::OutgoingResponse;
|
||||
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
||||
use crate::workspace::Workspace;
|
||||
use crate::workspace::hygiene::HygieneConfig;
|
||||
|
||||
/// Configuration for the heartbeat runner.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -96,6 +97,7 @@ pub enum HeartbeatResult {
|
||||
/// Heartbeat runner for proactive periodic execution.
|
||||
pub struct HeartbeatRunner {
|
||||
config: HeartbeatConfig,
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
|
||||
@@ -106,11 +108,13 @@ impl HeartbeatRunner {
|
||||
/// Create a new heartbeat runner.
|
||||
pub fn new(
|
||||
config: HeartbeatConfig,
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
) -> Self {
|
||||
Self {
|
||||
config,
|
||||
hygiene_config,
|
||||
workspace,
|
||||
llm,
|
||||
response_tx: None,
|
||||
@@ -145,6 +149,22 @@ impl HeartbeatRunner {
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
// Run memory hygiene in the background so it never delays the
|
||||
// heartbeat checklist. Failures are logged inside run_if_due.
|
||||
let hygiene_workspace = Arc::clone(&self.workspace);
|
||||
let hygiene_config = self.hygiene_config.clone();
|
||||
tokio::spawn(async move {
|
||||
let report =
|
||||
crate::workspace::hygiene::run_if_due(&hygiene_workspace, &hygiene_config)
|
||||
.await;
|
||||
if report.had_work() {
|
||||
tracing::info!(
|
||||
daily_logs_deleted = report.daily_logs_deleted,
|
||||
"heartbeat: memory hygiene deleted stale documents"
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
match self.check_heartbeat().await {
|
||||
HeartbeatResult::Ok => {
|
||||
tracing::debug!("Heartbeat OK");
|
||||
@@ -332,11 +352,12 @@ fn strip_html_comments(content: &str) -> String {
|
||||
/// Returns a handle that can be used to stop the runner.
|
||||
pub fn spawn_heartbeat(
|
||||
config: HeartbeatConfig,
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let mut runner = HeartbeatRunner::new(config, workspace, llm);
|
||||
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm);
|
||||
if let Some(tx) = response_tx {
|
||||
runner = runner.with_response_channel(tx);
|
||||
}
|
||||
|
||||
+1
-1
@@ -44,6 +44,6 @@ pub use self_repair::{BrokenTool, RepairResult, RepairTask, SelfRepair, StuckJob
|
||||
pub use session::{PendingApproval, PendingAuth, Session, Thread, ThreadState, Turn, TurnState};
|
||||
pub use session_manager::SessionManager;
|
||||
pub use submission::{Submission, SubmissionParser, SubmissionResult};
|
||||
pub use task::{Task, TaskContext, TaskHandler, TaskOutput, TaskStatus};
|
||||
pub use task::{Task, TaskContext, TaskHandler, TaskOutput};
|
||||
pub use undo::{Checkpoint, UndoManager};
|
||||
pub use worker::{Worker, WorkerDeps};
|
||||
|
||||
+38
-13
@@ -26,6 +26,8 @@ use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::RoutineError;
|
||||
|
||||
/// A routine is a named, persistent, user-owned task with a trigger and an action.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Routine {
|
||||
@@ -86,13 +88,16 @@ impl Trigger {
|
||||
}
|
||||
|
||||
/// Parse a trigger from its DB representation.
|
||||
pub fn from_db(trigger_type: &str, config: serde_json::Value) -> Result<Self, String> {
|
||||
pub fn from_db(trigger_type: &str, config: serde_json::Value) -> Result<Self, RoutineError> {
|
||||
match trigger_type {
|
||||
"cron" => {
|
||||
let schedule = config
|
||||
.get("schedule")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("cron trigger missing 'schedule'")?
|
||||
.ok_or_else(|| RoutineError::MissingField {
|
||||
context: "cron trigger".into(),
|
||||
field: "schedule".into(),
|
||||
})?
|
||||
.to_string();
|
||||
Ok(Trigger::Cron { schedule })
|
||||
}
|
||||
@@ -100,7 +105,10 @@ impl Trigger {
|
||||
let pattern = config
|
||||
.get("pattern")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("event trigger missing 'pattern'")?
|
||||
.ok_or_else(|| RoutineError::MissingField {
|
||||
context: "event trigger".into(),
|
||||
field: "pattern".into(),
|
||||
})?
|
||||
.to_string();
|
||||
let channel = config
|
||||
.get("channel")
|
||||
@@ -120,7 +128,9 @@ impl Trigger {
|
||||
Ok(Trigger::Webhook { path, secret })
|
||||
}
|
||||
"manual" => Ok(Trigger::Manual),
|
||||
other => Err(format!("unknown trigger type: {other}")),
|
||||
other => Err(RoutineError::UnknownTriggerType {
|
||||
trigger_type: other.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,13 +196,16 @@ impl RoutineAction {
|
||||
}
|
||||
|
||||
/// Parse an action from its DB representation.
|
||||
pub fn from_db(action_type: &str, config: serde_json::Value) -> Result<Self, String> {
|
||||
pub fn from_db(action_type: &str, config: serde_json::Value) -> Result<Self, RoutineError> {
|
||||
match action_type {
|
||||
"lightweight" => {
|
||||
let prompt = config
|
||||
.get("prompt")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("lightweight action missing 'prompt'")?
|
||||
.ok_or_else(|| RoutineError::MissingField {
|
||||
context: "lightweight action".into(),
|
||||
field: "prompt".into(),
|
||||
})?
|
||||
.to_string();
|
||||
let context_paths = config
|
||||
.get("context_paths")
|
||||
@@ -217,12 +230,18 @@ impl RoutineAction {
|
||||
let title = config
|
||||
.get("title")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("full_job action missing 'title'")?
|
||||
.ok_or_else(|| RoutineError::MissingField {
|
||||
context: "full_job action".into(),
|
||||
field: "title".into(),
|
||||
})?
|
||||
.to_string();
|
||||
let description = config
|
||||
.get("description")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("full_job action missing 'description'")?
|
||||
.ok_or_else(|| RoutineError::MissingField {
|
||||
context: "full_job action".into(),
|
||||
field: "description".into(),
|
||||
})?
|
||||
.to_string();
|
||||
let max_iterations = config
|
||||
.get("max_iterations")
|
||||
@@ -235,7 +254,9 @@ impl RoutineAction {
|
||||
max_iterations,
|
||||
})
|
||||
}
|
||||
other => Err(format!("unknown action type: {other}")),
|
||||
other => Err(RoutineError::UnknownActionType {
|
||||
action_type: other.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,14 +355,16 @@ impl std::fmt::Display for RunStatus {
|
||||
}
|
||||
|
||||
impl FromStr for RunStatus {
|
||||
type Err = String;
|
||||
type Err = RoutineError;
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"running" => Ok(RunStatus::Running),
|
||||
"ok" => Ok(RunStatus::Ok),
|
||||
"attention" => Ok(RunStatus::Attention),
|
||||
"failed" => Ok(RunStatus::Failed),
|
||||
other => Err(format!("unknown run status: {other}")),
|
||||
other => Err(RoutineError::UnknownRunStatus {
|
||||
status: other.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -370,9 +393,11 @@ pub fn content_hash(content: &str) -> u64 {
|
||||
}
|
||||
|
||||
/// Parse a cron expression and compute the next fire time from now.
|
||||
pub fn next_cron_fire(schedule: &str) -> Result<Option<DateTime<Utc>>, String> {
|
||||
pub fn next_cron_fire(schedule: &str) -> Result<Option<DateTime<Utc>>, RoutineError> {
|
||||
let cron_schedule =
|
||||
cron::Schedule::from_str(schedule).map_err(|e| format!("invalid cron: {e}"))?;
|
||||
cron::Schedule::from_str(schedule).map_err(|e| RoutineError::InvalidCron {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
Ok(cron_schedule.upcoming(Utc).next())
|
||||
}
|
||||
|
||||
|
||||
+58
-25
@@ -25,6 +25,7 @@ use crate::agent::routine::{
|
||||
use crate::channels::{IncomingMessage, OutgoingResponse};
|
||||
use crate::config::RoutineConfig;
|
||||
use crate::db::Database;
|
||||
use crate::error::RoutineError;
|
||||
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
||||
use crate::workspace::Workspace;
|
||||
|
||||
@@ -174,23 +175,26 @@ impl RoutineEngine {
|
||||
}
|
||||
|
||||
/// Fire a routine manually (from tool call or CLI).
|
||||
pub async fn fire_manual(&self, routine_id: Uuid) -> Result<Uuid, String> {
|
||||
pub async fn fire_manual(&self, routine_id: Uuid) -> Result<Uuid, RoutineError> {
|
||||
let routine = self
|
||||
.store
|
||||
.get_routine(routine_id)
|
||||
.await
|
||||
.map_err(|e| format!("DB error: {e}"))?
|
||||
.ok_or_else(|| format!("routine {routine_id} not found"))?;
|
||||
.map_err(|e| RoutineError::Database {
|
||||
reason: e.to_string(),
|
||||
})?
|
||||
.ok_or(RoutineError::NotFound { id: routine_id })?;
|
||||
|
||||
if !routine.enabled {
|
||||
return Err(format!("routine '{}' is disabled", routine.name));
|
||||
return Err(RoutineError::Disabled {
|
||||
name: routine.name.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
if !self.check_concurrent(&routine).await {
|
||||
return Err(format!(
|
||||
"routine '{}' already at max concurrent runs",
|
||||
routine.name
|
||||
));
|
||||
return Err(RoutineError::MaxConcurrent {
|
||||
name: routine.name.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let run_id = Uuid::new_v4();
|
||||
@@ -209,7 +213,9 @@ impl RoutineEngine {
|
||||
};
|
||||
|
||||
if let Err(e) = self.store.create_routine_run(&run).await {
|
||||
return Err(format!("failed to create run record: {e}"));
|
||||
return Err(RoutineError::Database {
|
||||
reason: format!("failed to create run record: {e}"),
|
||||
});
|
||||
}
|
||||
|
||||
// Execute inline for manual triggers (caller wants to wait)
|
||||
@@ -313,13 +319,27 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
|
||||
max_tokens,
|
||||
} => execute_lightweight(&ctx, &routine, prompt, context_paths, *max_tokens).await,
|
||||
RoutineAction::FullJob { description, .. } => {
|
||||
// Full job mode: for now, execute as lightweight with the description
|
||||
// as prompt. Full scheduler integration will come as a follow-up.
|
||||
tracing::info!(
|
||||
// Full job mode: scheduler integration not yet implemented.
|
||||
// Execute as lightweight and prepend a warning to the summary.
|
||||
tracing::warn!(
|
||||
routine = %routine.name,
|
||||
"FullJob mode executing as lightweight (scheduler integration pending)"
|
||||
"FullJob mode not yet implemented; falling back to lightweight execution"
|
||||
);
|
||||
execute_lightweight(&ctx, &routine, description, &[], ctx.max_lightweight_tokens).await
|
||||
match execute_lightweight(&ctx, &routine, description, &[], ctx.max_lightweight_tokens)
|
||||
.await
|
||||
{
|
||||
Ok((status, summary, tokens)) => {
|
||||
let warning = "[Note: FullJob mode is not yet implemented. This routine ran as \
|
||||
a single LLM call without tool access. Configure as 'lightweight' \
|
||||
or wait for full scheduler integration.]";
|
||||
let summary = match summary {
|
||||
Some(s) => Some(format!("{warning}\n\n{s}")),
|
||||
None => Some(warning.to_string()),
|
||||
};
|
||||
Ok((status, summary, tokens))
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -331,7 +351,7 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
|
||||
Ok(execution) => execution,
|
||||
Err(e) => {
|
||||
tracing::error!(routine = %routine.name, "Execution failed: {}", e);
|
||||
(RunStatus::Failed, Some(e), None)
|
||||
(RunStatus::Failed, Some(e.to_string()), None)
|
||||
}
|
||||
};
|
||||
|
||||
@@ -384,6 +404,20 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Sanitize a routine name for use in workspace paths.
|
||||
/// Only keeps alphanumeric, dash, and underscore characters; replaces everything else.
|
||||
fn sanitize_routine_name(name: &str) -> String {
|
||||
name.chars()
|
||||
.map(|c| {
|
||||
if c.is_ascii_alphanumeric() || c == '-' || c == '_' {
|
||||
c
|
||||
} else {
|
||||
'_'
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Execute a lightweight routine (single LLM call).
|
||||
async fn execute_lightweight(
|
||||
ctx: &EngineContext,
|
||||
@@ -391,7 +425,7 @@ async fn execute_lightweight(
|
||||
prompt: &str,
|
||||
context_paths: &[String],
|
||||
max_tokens: u32,
|
||||
) -> Result<(RunStatus, Option<String>, Option<i32>), String> {
|
||||
) -> Result<(RunStatus, Option<String>, Option<i32>), RoutineError> {
|
||||
// Load context from workspace
|
||||
let mut context_parts = Vec::new();
|
||||
for path in context_paths {
|
||||
@@ -408,8 +442,9 @@ async fn execute_lightweight(
|
||||
}
|
||||
}
|
||||
|
||||
// Load routine state from workspace
|
||||
let state_path = format!("routines/{}/state.md", routine.name);
|
||||
// Load routine state from workspace (name sanitized to prevent path traversal)
|
||||
let safe_name = sanitize_routine_name(&routine.name);
|
||||
let state_path = format!("routines/{safe_name}/state.md");
|
||||
let state_content = match ctx.workspace.read(&state_path).await {
|
||||
Ok(doc) => Some(doc.content),
|
||||
Err(_) => None,
|
||||
@@ -469,7 +504,9 @@ async fn execute_lightweight(
|
||||
.llm
|
||||
.complete(request)
|
||||
.await
|
||||
.map_err(|e| format!("LLM call failed: {e}"))?;
|
||||
.map_err(|e| RoutineError::LlmFailed {
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let content = response.content.trim();
|
||||
let tokens_used = Some((response.input_tokens + response.output_tokens) as i32);
|
||||
@@ -477,13 +514,9 @@ async fn execute_lightweight(
|
||||
// Empty content guard (same as heartbeat)
|
||||
if content.is_empty() {
|
||||
return if response.finish_reason == FinishReason::Length {
|
||||
Err(
|
||||
"LLM response truncated (finish_reason=length) with no content. \
|
||||
Model may have exhausted token budget on reasoning."
|
||||
.to_string(),
|
||||
)
|
||||
Err(RoutineError::TruncatedResponse)
|
||||
} else {
|
||||
Err("LLM returned empty content.".to_string())
|
||||
Err(RoutineError::EmptyResponse)
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+11
-3
@@ -136,7 +136,9 @@ impl Scheduler {
|
||||
});
|
||||
|
||||
// Start the worker
|
||||
let _ = tx.send(WorkerMessage::Start).await;
|
||||
if tx.send(WorkerMessage::Start).await.is_err() {
|
||||
tracing::error!(job_id = %job_id, "Worker died before receiving Start message");
|
||||
}
|
||||
|
||||
// Insert while still holding the write lock
|
||||
jobs.insert(job_id, ScheduledJob { handle, tx });
|
||||
@@ -418,10 +420,16 @@ impl Scheduler {
|
||||
// Update job state
|
||||
self.context_manager
|
||||
.update_context(job_id, |ctx| {
|
||||
let _ = ctx.transition_to(
|
||||
if let Err(e) = ctx.transition_to(
|
||||
JobState::Cancelled,
|
||||
Some("Stopped by scheduler".to_string()),
|
||||
);
|
||||
) {
|
||||
tracing::warn!(
|
||||
job_id = %job_id,
|
||||
error = %e,
|
||||
"Failed to transition job to Cancelled state"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -66,12 +66,14 @@ pub trait SelfRepair: Send + Sync {
|
||||
/// Default self-repair implementation.
|
||||
pub struct DefaultSelfRepair {
|
||||
context_manager: Arc<ContextManager>,
|
||||
#[allow(dead_code)] // Will be used for time-based stuck detection
|
||||
// TODO: use for time-based stuck detection (currently only max_repair_attempts is checked)
|
||||
#[allow(dead_code)]
|
||||
stuck_threshold: Duration,
|
||||
max_repair_attempts: u32,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
builder: Option<Arc<dyn SoftwareBuilder>>,
|
||||
#[allow(dead_code)] // Will be used for tool hot-reload after repair
|
||||
// TODO: use for tool hot-reload after repair
|
||||
#[allow(dead_code)]
|
||||
tools: Option<Arc<ToolRegistry>>,
|
||||
}
|
||||
|
||||
@@ -93,15 +95,15 @@ impl DefaultSelfRepair {
|
||||
}
|
||||
|
||||
/// Add a Store for tool failure tracking.
|
||||
#[allow(dead_code)] // Public API for configuring repair with persistence
|
||||
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||
#[allow(dead_code)] // TODO: wire up in main.rs when persistence is needed
|
||||
pub(crate) fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||
self.store = Some(store);
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a Builder and ToolRegistry for automatic tool repair.
|
||||
#[allow(dead_code)] // Public API for enabling automatic tool repair
|
||||
pub fn with_builder(
|
||||
#[allow(dead_code)] // TODO: wire up in main.rs when auto-repair is needed
|
||||
pub(crate) fn with_builder(
|
||||
mut self,
|
||||
builder: Arc<dyn SoftwareBuilder>,
|
||||
tools: Arc<ToolRegistry>,
|
||||
|
||||
+19
-8
@@ -70,10 +70,9 @@ impl Session {
|
||||
pub fn create_thread(&mut self) -> &mut Thread {
|
||||
let thread = Thread::new(self.id);
|
||||
let thread_id = thread.id;
|
||||
self.threads.insert(thread_id, thread);
|
||||
self.active_thread = Some(thread_id);
|
||||
self.last_active_at = Utc::now();
|
||||
self.threads.get_mut(&thread_id).expect("just inserted")
|
||||
self.threads.entry(thread_id).or_insert(thread)
|
||||
}
|
||||
|
||||
/// Get the active thread.
|
||||
@@ -88,10 +87,19 @@ impl Session {
|
||||
|
||||
/// Get or create the active thread.
|
||||
pub fn get_or_create_thread(&mut self) -> &mut Thread {
|
||||
if self.active_thread.is_none() {
|
||||
self.create_thread();
|
||||
match self.active_thread {
|
||||
None => self.create_thread(),
|
||||
Some(id) => {
|
||||
if self.threads.contains_key(&id) {
|
||||
// Safe: contains_key confirmed the entry exists.
|
||||
self.threads.get_mut(&id).unwrap()
|
||||
} else {
|
||||
// Stale active_thread ID: create a new thread, which
|
||||
// updates self.active_thread to the new thread's ID.
|
||||
self.create_thread()
|
||||
}
|
||||
}
|
||||
}
|
||||
self.active_thread_mut().expect("just created")
|
||||
}
|
||||
|
||||
/// Switch to a different thread.
|
||||
@@ -240,7 +248,8 @@ impl Thread {
|
||||
self.turns.push(turn);
|
||||
self.state = ThreadState::Processing;
|
||||
self.updated_at = Utc::now();
|
||||
self.turns.last_mut().expect("just pushed")
|
||||
// turn_number was len() before push, so it's a valid index after push
|
||||
&mut self.turns[turn_number]
|
||||
}
|
||||
|
||||
/// Complete the current turn with a response.
|
||||
@@ -353,8 +362,10 @@ impl Thread {
|
||||
if let Some(next) = iter.peek()
|
||||
&& next.role == crate::llm::Role::Assistant
|
||||
{
|
||||
let response = iter.next().expect("peeked");
|
||||
turn.complete(&response.content);
|
||||
// iter.next() is guaranteed Some after a successful peek()
|
||||
if let Some(response) = iter.next() {
|
||||
turn.complete(&response.content);
|
||||
}
|
||||
}
|
||||
|
||||
self.turns.push(turn);
|
||||
|
||||
@@ -13,6 +13,9 @@ use crate::agent::session::Session;
|
||||
use crate::agent::undo::UndoManager;
|
||||
use crate::hooks::HookRegistry;
|
||||
|
||||
/// Warn when session count exceeds this threshold.
|
||||
const SESSION_COUNT_WARNING_THRESHOLD: usize = 1000;
|
||||
|
||||
/// Key for mapping external thread IDs to internal ones.
|
||||
#[derive(Clone, Hash, Eq, PartialEq)]
|
||||
struct ThreadKey {
|
||||
@@ -68,6 +71,14 @@ impl SessionManager {
|
||||
let session = Arc::new(Mutex::new(new_session));
|
||||
sessions.insert(user_id.to_string(), Arc::clone(&session));
|
||||
|
||||
if sessions.len() >= SESSION_COUNT_WARNING_THRESHOLD && sessions.len() % 100 == 0 {
|
||||
tracing::warn!(
|
||||
"High session count: {} active sessions. \
|
||||
Pruning runs every 10 minutes; consider reducing session_idle_timeout.",
|
||||
sessions.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Fire OnSessionStart hook (fire-and-forget)
|
||||
if let Some(ref hooks) = self.hooks {
|
||||
let hooks = hooks.clone();
|
||||
|
||||
@@ -234,6 +234,7 @@ impl Submission {
|
||||
}
|
||||
|
||||
/// Create an approval submission.
|
||||
#[cfg(test)]
|
||||
pub fn approval(request_id: Uuid, approved: bool) -> Self {
|
||||
Self::ExecApproval {
|
||||
request_id,
|
||||
@@ -243,6 +244,7 @@ impl Submission {
|
||||
}
|
||||
|
||||
/// Create an "always approve" submission.
|
||||
#[cfg(test)]
|
||||
pub fn always_approve(request_id: Uuid) -> Self {
|
||||
Self::ExecApproval {
|
||||
request_id,
|
||||
@@ -252,26 +254,31 @@ impl Submission {
|
||||
}
|
||||
|
||||
/// Create an interrupt submission.
|
||||
#[cfg(test)]
|
||||
pub fn interrupt() -> Self {
|
||||
Self::Interrupt
|
||||
}
|
||||
|
||||
/// Create a compact submission.
|
||||
#[cfg(test)]
|
||||
pub fn compact() -> Self {
|
||||
Self::Compact
|
||||
}
|
||||
|
||||
/// Create an undo submission.
|
||||
#[cfg(test)]
|
||||
pub fn undo() -> Self {
|
||||
Self::Undo
|
||||
}
|
||||
|
||||
/// Create a redo submission.
|
||||
#[cfg(test)]
|
||||
pub fn redo() -> Self {
|
||||
Self::Redo
|
||||
}
|
||||
|
||||
/// Check if this submission starts a new turn.
|
||||
#[cfg(test)]
|
||||
pub fn starts_turn(&self) -> bool {
|
||||
matches!(self, Self::UserInput { .. })
|
||||
}
|
||||
@@ -340,6 +347,7 @@ impl SubmissionResult {
|
||||
}
|
||||
|
||||
/// Create an OK result.
|
||||
#[cfg(test)]
|
||||
pub fn ok() -> Self {
|
||||
Self::Ok { message: None }
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ impl TaskOutput {
|
||||
}
|
||||
|
||||
/// Create a text result.
|
||||
#[cfg(test)]
|
||||
pub fn text(text: impl Into<String>, duration: Duration) -> Self {
|
||||
Self {
|
||||
result: serde_json::Value::String(text.into()),
|
||||
@@ -37,6 +38,7 @@ impl TaskOutput {
|
||||
}
|
||||
|
||||
/// Create an empty success result.
|
||||
#[cfg(test)]
|
||||
pub fn empty(duration: Duration) -> Self {
|
||||
Self {
|
||||
result: serde_json::Value::Null,
|
||||
@@ -130,6 +132,7 @@ impl Task {
|
||||
}
|
||||
|
||||
/// Create a new Job task with a specific ID.
|
||||
#[cfg(test)]
|
||||
pub fn job_with_id(id: Uuid, title: impl Into<String>, description: impl Into<String>) -> Self {
|
||||
Self::Job {
|
||||
id,
|
||||
@@ -152,6 +155,7 @@ impl Task {
|
||||
}
|
||||
|
||||
/// Create a new Background task.
|
||||
#[cfg(test)]
|
||||
pub fn background(handler: std::sync::Arc<dyn TaskHandler>) -> Self {
|
||||
Self::Background {
|
||||
id: Uuid::new_v4(),
|
||||
@@ -160,6 +164,7 @@ impl Task {
|
||||
}
|
||||
|
||||
/// Create a new Background task with a specific ID.
|
||||
#[cfg(test)]
|
||||
pub fn background_with_id(id: Uuid, handler: std::sync::Arc<dyn TaskHandler>) -> Self {
|
||||
Self::Background { id, handler }
|
||||
}
|
||||
@@ -174,6 +179,7 @@ impl Task {
|
||||
}
|
||||
|
||||
/// Get the parent ID for sub-tasks.
|
||||
#[cfg(test)]
|
||||
pub fn parent_id(&self) -> Option<Uuid> {
|
||||
match self {
|
||||
Self::Job { .. } => None,
|
||||
@@ -225,6 +231,7 @@ impl fmt::Debug for Task {
|
||||
}
|
||||
|
||||
/// Status of a scheduled task.
|
||||
#[cfg(test)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TaskStatus {
|
||||
/// Task is queued waiting for execution.
|
||||
|
||||
+280
-122
@@ -6,11 +6,14 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::task::JoinSet;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::agent::Agent;
|
||||
use crate::agent::compaction::ContextCompactor;
|
||||
use crate::agent::dispatcher::{AgenticLoopResult, detect_auth_awaiting, parse_auth_result};
|
||||
use crate::agent::dispatcher::{
|
||||
AgenticLoopResult, check_auth_required, execute_chat_tool_standalone, parse_auth_result,
|
||||
};
|
||||
use crate::agent::session::{PendingApproval, Session, ThreadState};
|
||||
use crate::agent::submission::SubmissionResult;
|
||||
use crate::channels::{IncomingMessage, StatusUpdate};
|
||||
@@ -608,8 +611,8 @@ impl Agent {
|
||||
approved: bool,
|
||||
always: bool,
|
||||
) -> Result<SubmissionResult, Error> {
|
||||
// Get thread state and pending approval
|
||||
let (_thread_state, pending) = {
|
||||
// Get pending approval for this thread
|
||||
let pending = {
|
||||
let mut sess = session.lock().await;
|
||||
let thread = sess
|
||||
.threads
|
||||
@@ -620,8 +623,7 @@ impl Agent {
|
||||
return Ok(SubmissionResult::error("No pending approval request."));
|
||||
}
|
||||
|
||||
let pending = thread.take_pending_approval();
|
||||
(thread.state, pending)
|
||||
thread.take_pending_approval()
|
||||
};
|
||||
|
||||
let pending = match pending {
|
||||
@@ -734,29 +736,17 @@ impl Agent {
|
||||
// If tool_auth returned awaiting_token, enter auth mode and
|
||||
// return instructions directly (skip agentic loop continuation).
|
||||
if let Some((ext_name, instructions)) =
|
||||
detect_auth_awaiting(&pending.tool_name, &tool_result)
|
||||
check_auth_required(&pending.tool_name, &tool_result)
|
||||
{
|
||||
let auth_data = parse_auth_result(&tool_result);
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
thread.enter_auth_mode(ext_name.clone());
|
||||
thread.complete_turn(&instructions);
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::AuthRequired {
|
||||
extension_name: ext_name,
|
||||
instructions: Some(instructions.clone()),
|
||||
auth_url: auth_data.auth_url,
|
||||
setup_url: auth_data.setup_url,
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
self.handle_auth_intercept(
|
||||
&session,
|
||||
thread_id,
|
||||
message,
|
||||
&tool_result,
|
||||
ext_name,
|
||||
instructions.clone(),
|
||||
)
|
||||
.await;
|
||||
return Ok(SubmissionResult::response(instructions));
|
||||
}
|
||||
|
||||
@@ -798,9 +788,17 @@ impl Agent {
|
||||
.await;
|
||||
}
|
||||
|
||||
let mut deferred_queue = std::collections::VecDeque::from(deferred_tool_calls);
|
||||
while let Some(tc) = deferred_queue.pop_front() {
|
||||
// Re-check approval for each deferred tool call
|
||||
// === Phase 1: Preflight (sequential) ===
|
||||
// Walk deferred tools checking approval. Collect runnable
|
||||
// tools; stop at the first that needs approval.
|
||||
let mut runnable: Vec<crate::llm::ToolCall> = Vec::new();
|
||||
let mut approval_needed: Option<(
|
||||
usize,
|
||||
crate::llm::ToolCall,
|
||||
Arc<dyn crate::tools::Tool>,
|
||||
)> = None;
|
||||
|
||||
for (idx, tc) in deferred_tool_calls.iter().enumerate() {
|
||||
if let Some(tool) = self.tools().get(&tc.name).await
|
||||
&& tool.requires_approval()
|
||||
{
|
||||
@@ -814,73 +812,142 @@ impl Agent {
|
||||
};
|
||||
|
||||
if !is_auto_approved {
|
||||
let new_pending = PendingApproval {
|
||||
request_id: Uuid::new_v4(),
|
||||
tool_name: tc.name.clone(),
|
||||
parameters: tc.arguments.clone(),
|
||||
description: tool.description().to_string(),
|
||||
tool_call_id: tc.id.clone(),
|
||||
context_messages: context_messages.clone(),
|
||||
deferred_tool_calls: deferred_queue.iter().cloned().collect(),
|
||||
};
|
||||
|
||||
let request_id = new_pending.request_id;
|
||||
let tool_name = new_pending.tool_name.clone();
|
||||
let description = new_pending.description.clone();
|
||||
let parameters = new_pending.parameters.clone();
|
||||
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
thread.await_approval(new_pending);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::Status("Awaiting approval".into()),
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(SubmissionResult::NeedApproval {
|
||||
request_id,
|
||||
tool_name,
|
||||
description,
|
||||
parameters,
|
||||
});
|
||||
approval_needed = Some((idx, tc.clone(), tool));
|
||||
break; // remaining tools stay deferred
|
||||
}
|
||||
}
|
||||
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::ToolStarted {
|
||||
name: tc.name.clone(),
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
runnable.push(tc.clone());
|
||||
}
|
||||
|
||||
let deferred_result = self
|
||||
.execute_chat_tool(&tc.name, &tc.arguments, &job_ctx)
|
||||
.await;
|
||||
// === Phase 2: Parallel execution ===
|
||||
let exec_results: Vec<(crate::llm::ToolCall, Result<String, Error>)> = if runnable.len()
|
||||
<= 1
|
||||
{
|
||||
// Single tool (or none): execute inline
|
||||
let mut results = Vec::new();
|
||||
for tc in &runnable {
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::ToolStarted {
|
||||
name: tc.name.clone(),
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::ToolCompleted {
|
||||
name: tc.name.clone(),
|
||||
success: deferred_result.is_ok(),
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
let result = self
|
||||
.execute_chat_tool(&tc.name, &tc.arguments, &job_ctx)
|
||||
.await;
|
||||
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::ToolCompleted {
|
||||
name: tc.name.clone(),
|
||||
success: result.is_ok(),
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
results.push((tc.clone(), result));
|
||||
}
|
||||
results
|
||||
} else {
|
||||
// Multiple tools: execute in parallel via JoinSet
|
||||
let mut join_set = JoinSet::new();
|
||||
let runnable_count = runnable.len();
|
||||
|
||||
for (spawn_idx, tc) in runnable.iter().enumerate() {
|
||||
let tools = self.tools().clone();
|
||||
let safety = self.safety().clone();
|
||||
let channels = self.channels.clone();
|
||||
let job_ctx = job_ctx.clone();
|
||||
let tc = tc.clone();
|
||||
let channel = message.channel.clone();
|
||||
let metadata = message.metadata.clone();
|
||||
|
||||
join_set.spawn(async move {
|
||||
let _ = channels
|
||||
.send_status(
|
||||
&channel,
|
||||
StatusUpdate::ToolStarted {
|
||||
name: tc.name.clone(),
|
||||
},
|
||||
&metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
let result = execute_chat_tool_standalone(
|
||||
&tools,
|
||||
&safety,
|
||||
&tc.name,
|
||||
&tc.arguments,
|
||||
&job_ctx,
|
||||
)
|
||||
.await;
|
||||
|
||||
let _ = channels
|
||||
.send_status(
|
||||
&channel,
|
||||
StatusUpdate::ToolCompleted {
|
||||
name: tc.name.clone(),
|
||||
success: result.is_ok(),
|
||||
},
|
||||
&metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
(spawn_idx, tc, result)
|
||||
});
|
||||
}
|
||||
|
||||
// Collect and reorder by original index
|
||||
let mut ordered: Vec<Option<(crate::llm::ToolCall, Result<String, Error>)>> =
|
||||
(0..runnable_count).map(|_| None).collect();
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
match join_result {
|
||||
Ok((idx, tc, result)) => {
|
||||
ordered[idx] = Some((tc, result));
|
||||
}
|
||||
Err(e) => {
|
||||
if e.is_panic() {
|
||||
tracing::error!("Deferred tool execution task panicked: {}", e);
|
||||
} else {
|
||||
tracing::error!("Deferred tool execution task cancelled: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fill panicked slots with error results
|
||||
ordered
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, opt)| {
|
||||
opt.unwrap_or_else(|| {
|
||||
let tc = runnable[i].clone();
|
||||
let err: Error = crate::error::ToolError::ExecutionFailed {
|
||||
name: tc.name.clone(),
|
||||
reason: "Task failed during execution".to_string(),
|
||||
}
|
||||
.into();
|
||||
(tc, Err(err))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
// === Phase 3: Post-flight (sequential, in original order) ===
|
||||
// Process all results before any conditional return so every
|
||||
// tool result is recorded in the session audit trail.
|
||||
let mut deferred_auth: Option<String> = None;
|
||||
|
||||
for (tc, deferred_result) in exec_results {
|
||||
if let Ok(ref output) = deferred_result
|
||||
&& !output.is_empty()
|
||||
{
|
||||
@@ -910,32 +977,21 @@ impl Agent {
|
||||
}
|
||||
}
|
||||
|
||||
// Auth detection for deferred tools
|
||||
if let Some((ext_name, instructions)) =
|
||||
detect_auth_awaiting(&tc.name, &deferred_result)
|
||||
// Auth detection — defer return until all results are recorded
|
||||
if deferred_auth.is_none()
|
||||
&& let Some((ext_name, instructions)) =
|
||||
check_auth_required(&tc.name, &deferred_result)
|
||||
{
|
||||
let auth_data = parse_auth_result(&deferred_result);
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
thread.enter_auth_mode(ext_name.clone());
|
||||
thread.complete_turn(&instructions);
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::AuthRequired {
|
||||
extension_name: ext_name,
|
||||
instructions: Some(instructions.clone()),
|
||||
auth_url: auth_data.auth_url,
|
||||
setup_url: auth_data.setup_url,
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
return Ok(SubmissionResult::response(instructions));
|
||||
self.handle_auth_intercept(
|
||||
&session,
|
||||
thread_id,
|
||||
message,
|
||||
&deferred_result,
|
||||
ext_name,
|
||||
instructions.clone(),
|
||||
)
|
||||
.await;
|
||||
deferred_auth = Some(instructions);
|
||||
}
|
||||
|
||||
let deferred_content = match deferred_result {
|
||||
@@ -953,6 +1009,52 @@ impl Agent {
|
||||
context_messages.push(ChatMessage::tool_result(&tc.id, &tc.name, deferred_content));
|
||||
}
|
||||
|
||||
// Return auth response after all results are recorded
|
||||
if let Some(instructions) = deferred_auth {
|
||||
return Ok(SubmissionResult::response(instructions));
|
||||
}
|
||||
|
||||
// Handle approval if a tool needed it
|
||||
if let Some((approval_idx, tc, tool)) = approval_needed {
|
||||
let new_pending = PendingApproval {
|
||||
request_id: Uuid::new_v4(),
|
||||
tool_name: tc.name.clone(),
|
||||
parameters: tc.arguments.clone(),
|
||||
description: tool.description().to_string(),
|
||||
tool_call_id: tc.id.clone(),
|
||||
context_messages: context_messages.clone(),
|
||||
deferred_tool_calls: deferred_tool_calls[approval_idx + 1..].to_vec(),
|
||||
};
|
||||
|
||||
let request_id = new_pending.request_id;
|
||||
let tool_name = new_pending.tool_name.clone();
|
||||
let description = new_pending.description.clone();
|
||||
let parameters = new_pending.parameters.clone();
|
||||
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
thread.await_approval(new_pending);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::Status("Awaiting approval".into()),
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
|
||||
return Ok(SubmissionResult::NeedApproval {
|
||||
request_id,
|
||||
tool_name,
|
||||
description,
|
||||
parameters,
|
||||
});
|
||||
}
|
||||
|
||||
// Continue the agentic loop (a tool was already executed this turn)
|
||||
let result = self
|
||||
.run_agentic_loop(message, session.clone(), thread_id, context_messages, true)
|
||||
@@ -967,7 +1069,11 @@ impl Agent {
|
||||
|
||||
match result {
|
||||
Ok(AgenticLoopResult::Response(response)) => {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.complete_turn(&response);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&response));
|
||||
}
|
||||
self.persist_response_chain(thread);
|
||||
let _ = self
|
||||
.channels
|
||||
@@ -1003,16 +1109,30 @@ impl Agent {
|
||||
})
|
||||
}
|
||||
Err(e) => {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.fail_turn(e.to_string());
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, None);
|
||||
}
|
||||
Ok(SubmissionResult::error(e.to_string()))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Rejected - clear approval and return to idle
|
||||
// Rejected - complete the turn with a rejection message and persist
|
||||
let rejection = format!(
|
||||
"Tool '{}' was rejected. The agent will not execute this tool.\n\n\
|
||||
You can continue the conversation or try a different approach.",
|
||||
pending.tool_name
|
||||
);
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.clear_pending_approval();
|
||||
thread.complete_turn(&rejection);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&rejection));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1025,14 +1145,52 @@ impl Agent {
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(SubmissionResult::response(format!(
|
||||
"Tool '{}' was rejected. The agent will not execute this tool.\n\n\
|
||||
You can continue the conversation or try a different approach.",
|
||||
pending.tool_name
|
||||
)))
|
||||
Ok(SubmissionResult::response(rejection))
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle an auth-required result from a tool execution.
|
||||
///
|
||||
/// Enters auth mode on the thread, completes + persists the turn,
|
||||
/// and sends the AuthRequired status to the channel.
|
||||
/// Returns the instructions string for the caller to wrap in a response.
|
||||
async fn handle_auth_intercept(
|
||||
&self,
|
||||
session: &Arc<Mutex<Session>>,
|
||||
thread_id: Uuid,
|
||||
message: &IncomingMessage,
|
||||
tool_result: &Result<String, Error>,
|
||||
ext_name: String,
|
||||
instructions: String,
|
||||
) {
|
||||
let auth_data = parse_auth_result(tool_result);
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.enter_auth_mode(ext_name.clone());
|
||||
thread.complete_turn(&instructions);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&instructions));
|
||||
}
|
||||
self.persist_response_chain(thread);
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::AuthRequired {
|
||||
extension_name: ext_name,
|
||||
instructions: Some(instructions.clone()),
|
||||
auth_url: auth_data.auth_url,
|
||||
setup_url: auth_data.setup_url,
|
||||
},
|
||||
&message.metadata,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Handle an auth token submitted while the thread is in auth mode.
|
||||
///
|
||||
/// The token goes directly to the extension manager's credential store,
|
||||
|
||||
@@ -67,6 +67,7 @@ impl UndoManager {
|
||||
}
|
||||
|
||||
/// Create with a custom checkpoint limit.
|
||||
#[cfg(test)]
|
||||
pub fn with_max_checkpoints(mut self, max: usize) -> Self {
|
||||
self.max_checkpoints = max;
|
||||
self
|
||||
@@ -126,6 +127,7 @@ impl UndoManager {
|
||||
}
|
||||
|
||||
/// Pop the last checkpoint from the undo stack.
|
||||
#[cfg(test)]
|
||||
pub fn pop_undo(&mut self) -> Option<Checkpoint> {
|
||||
self.undo_stack.pop_back()
|
||||
}
|
||||
@@ -178,6 +180,7 @@ impl UndoManager {
|
||||
}
|
||||
|
||||
/// Get a checkpoint by ID.
|
||||
#[cfg(test)]
|
||||
pub fn get_checkpoint(&self, id: Uuid) -> Option<&Checkpoint> {
|
||||
self.undo_stack
|
||||
.iter()
|
||||
@@ -186,6 +189,7 @@ impl UndoManager {
|
||||
}
|
||||
|
||||
/// List all available checkpoints (for UI display).
|
||||
#[cfg(test)]
|
||||
pub fn list_checkpoints(&self) -> Vec<&Checkpoint> {
|
||||
self.undo_stack.iter().collect()
|
||||
}
|
||||
|
||||
+329
-46
@@ -3,8 +3,8 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::join_all;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::agent::scheduler::WorkerMessage;
|
||||
@@ -292,19 +292,21 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
tool_calls.clone(),
|
||||
));
|
||||
|
||||
for tc in tool_calls {
|
||||
let result = self.execute_tool(&tc.name, &tc.arguments).await;
|
||||
|
||||
// Create synthetic selection for process_tool_result
|
||||
let selection = ToolSelection {
|
||||
// Convert ToolCalls to ToolSelections and execute in parallel
|
||||
let selections: Vec<ToolSelection> = tool_calls
|
||||
.iter()
|
||||
.map(|tc| ToolSelection {
|
||||
tool_name: tc.name.clone(),
|
||||
parameters: tc.arguments.clone(),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: tc.id.clone(),
|
||||
};
|
||||
})
|
||||
.collect();
|
||||
|
||||
self.process_tool_result(reason_ctx, &selection, result)
|
||||
let results = self.execute_tools_parallel(&selections).await;
|
||||
for (selection, result) in selections.iter().zip(results) {
|
||||
self.process_tool_result(reason_ctx, selection, result.result)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
@@ -347,24 +349,71 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute multiple tools in parallel.
|
||||
/// Execute multiple tools in parallel using a JoinSet.
|
||||
///
|
||||
/// Each task is tagged with its original index so results are returned
|
||||
/// in the same order as `selections`, regardless of completion order.
|
||||
async fn execute_tools_parallel(&self, selections: &[ToolSelection]) -> Vec<ToolExecResult> {
|
||||
let futures: Vec<_> = selections
|
||||
.iter()
|
||||
.map(|selection| {
|
||||
let tool_name = selection.tool_name.clone();
|
||||
let params = selection.parameters.clone();
|
||||
let deps = self.deps.clone();
|
||||
let job_id = self.job_id;
|
||||
let count = selections.len();
|
||||
|
||||
async move {
|
||||
let result = Self::execute_tool_inner(&deps, job_id, &tool_name, ¶ms).await;
|
||||
ToolExecResult { result }
|
||||
// Short-circuit for single tool: execute directly without JoinSet overhead
|
||||
if count <= 1 {
|
||||
let mut results = Vec::with_capacity(count);
|
||||
for selection in selections {
|
||||
let result = Self::execute_tool_inner(
|
||||
&self.deps,
|
||||
self.job_id,
|
||||
&selection.tool_name,
|
||||
&selection.parameters,
|
||||
)
|
||||
.await;
|
||||
results.push(ToolExecResult { result });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
for (idx, selection) in selections.iter().enumerate() {
|
||||
let deps = self.deps.clone();
|
||||
let job_id = self.job_id;
|
||||
let tool_name = selection.tool_name.clone();
|
||||
let params = selection.parameters.clone();
|
||||
join_set.spawn(async move {
|
||||
let result = Self::execute_tool_inner(&deps, job_id, &tool_name, ¶ms).await;
|
||||
(idx, ToolExecResult { result })
|
||||
});
|
||||
}
|
||||
|
||||
// Collect and reorder by original index
|
||||
let mut results: Vec<Option<ToolExecResult>> = (0..count).map(|_| None).collect();
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
match join_result {
|
||||
Ok((idx, exec_result)) => results[idx] = Some(exec_result),
|
||||
Err(e) => {
|
||||
if e.is_panic() {
|
||||
tracing::error!("Tool execution task panicked: {}", e);
|
||||
} else {
|
||||
tracing::error!("Tool execution task cancelled: {}", e);
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
}
|
||||
|
||||
join_all(futures).await
|
||||
// Fill any panicked slots with error results
|
||||
results
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, opt)| {
|
||||
opt.unwrap_or_else(|| ToolExecResult {
|
||||
result: Err(crate::error::ToolError::ExecutionFailed {
|
||||
name: selections[i].tool_name.clone(),
|
||||
reason: "Task failed during execution".to_string(),
|
||||
}
|
||||
.into()),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Inner tool execution logic that can be called from both single and parallel paths.
|
||||
@@ -505,7 +554,8 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
let output_str = serde_json::to_string_pretty(&output.result)
|
||||
.ok()
|
||||
.map(|s| deps.safety.sanitize_tool_output(tool_name, &s).content);
|
||||
deps.context_manager
|
||||
match deps
|
||||
.context_manager
|
||||
.update_memory(job_id, |mem| {
|
||||
let rec = mem.create_action(tool_name, params.clone()).succeed(
|
||||
output_str.clone(),
|
||||
@@ -516,30 +566,52 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
rec
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
{
|
||||
Ok(rec) => Some(rec),
|
||||
Err(e) => {
|
||||
tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
match deps
|
||||
.context_manager
|
||||
.update_memory(job_id, |mem| {
|
||||
let rec = mem
|
||||
.create_action(tool_name, params.clone())
|
||||
.fail(e.to_string(), elapsed);
|
||||
mem.record_action(rec.clone());
|
||||
rec
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(rec) => Some(rec),
|
||||
Err(e) => {
|
||||
tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
match deps
|
||||
.context_manager
|
||||
.update_memory(job_id, |mem| {
|
||||
let rec = mem
|
||||
.create_action(tool_name, params.clone())
|
||||
.fail("Execution timeout", elapsed);
|
||||
mem.record_action(rec.clone());
|
||||
rec
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(rec) => Some(rec),
|
||||
Err(e) => {
|
||||
tracing::warn!(job_id = %job_id, tool = tool_name, "Failed to record action in memory: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => deps
|
||||
.context_manager
|
||||
.update_memory(job_id, |mem| {
|
||||
let rec = mem
|
||||
.create_action(tool_name, params.clone())
|
||||
.fail(e.to_string(), elapsed);
|
||||
mem.record_action(rec.clone());
|
||||
rec
|
||||
})
|
||||
.await
|
||||
.ok(),
|
||||
Err(_) => deps
|
||||
.context_manager
|
||||
.update_memory(job_id, |mem| {
|
||||
let rec = mem
|
||||
.create_action(tool_name, params.clone())
|
||||
.fail("Execution timeout", elapsed);
|
||||
mem.record_action(rec.clone());
|
||||
rec
|
||||
})
|
||||
.await
|
||||
.ok(),
|
||||
};
|
||||
|
||||
// Persist action to database (fire-and-forget)
|
||||
@@ -800,6 +872,102 @@ mod tests {
|
||||
use crate::llm::ToolSelection;
|
||||
use crate::util::llm_signals_completion;
|
||||
|
||||
use super::*;
|
||||
use crate::config::SafetyConfig;
|
||||
use crate::context::JobContext;
|
||||
use crate::llm::{
|
||||
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::tools::{Tool, ToolError, ToolOutput};
|
||||
|
||||
/// A test tool that sleeps for a configurable duration before returning.
|
||||
struct SlowTool {
|
||||
tool_name: String,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Tool for SlowTool {
|
||||
fn name(&self) -> &str {
|
||||
&self.tool_name
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"Test tool with configurable delay"
|
||||
}
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
serde_json::json!({"type": "object", "properties": {}})
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_params: serde_json::Value,
|
||||
_ctx: &JobContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let start = std::time::Instant::now();
|
||||
tokio::time::sleep(self.delay).await;
|
||||
Ok(ToolOutput::text(
|
||||
format!("done_{}", self.tool_name),
|
||||
start.elapsed(),
|
||||
))
|
||||
}
|
||||
fn requires_sanitization(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Stub LLM provider (never called in these tests).
|
||||
struct StubLlm;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl LlmProvider for StubLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
"stub"
|
||||
}
|
||||
fn cost_per_token(&self) -> (rust_decimal::Decimal, rust_decimal::Decimal) {
|
||||
(rust_decimal::Decimal::ZERO, rust_decimal::Decimal::ZERO)
|
||||
}
|
||||
async fn complete(
|
||||
&self,
|
||||
_req: CompletionRequest,
|
||||
) -> Result<CompletionResponse, crate::error::LlmError> {
|
||||
unimplemented!("stub")
|
||||
}
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
_req: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, crate::error::LlmError> {
|
||||
unimplemented!("stub")
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a Worker wired to a ToolRegistry containing the given tools.
|
||||
async fn make_worker(tools: Vec<Arc<dyn Tool>>) -> Worker {
|
||||
let registry = ToolRegistry::new();
|
||||
for t in tools {
|
||||
registry.register(t).await;
|
||||
}
|
||||
|
||||
let cm = Arc::new(crate::context::ContextManager::new(5));
|
||||
let job_id = cm.create_job("test", "test job").await.unwrap();
|
||||
|
||||
let deps = WorkerDeps {
|
||||
context_manager: cm,
|
||||
llm: Arc::new(StubLlm),
|
||||
safety: Arc::new(SafetyLayer::new(&SafetyConfig {
|
||||
max_output_length: 100_000,
|
||||
injection_check_enabled: false,
|
||||
})),
|
||||
tools: Arc::new(registry),
|
||||
store: None,
|
||||
hooks: Arc::new(crate::hooks::HookRegistry::new()),
|
||||
timeout: Duration::from_secs(30),
|
||||
use_planning: false,
|
||||
};
|
||||
|
||||
Worker::new(job_id, deps)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tool_selection_preserves_call_id() {
|
||||
let selection = ToolSelection {
|
||||
@@ -876,4 +1044,119 @@ mod tests {
|
||||
"The tool returned: TASK_COMPLETE signal"
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_parallel_speedup() {
|
||||
// 3 tools each sleeping 200ms should finish in roughly 200ms (parallel),
|
||||
// not ~600ms (sequential).
|
||||
let tools: Vec<Arc<dyn Tool>> = (0..3)
|
||||
.map(|i| {
|
||||
Arc::new(SlowTool {
|
||||
tool_name: format!("slow_{}", i),
|
||||
delay: Duration::from_millis(200),
|
||||
}) as Arc<dyn Tool>
|
||||
})
|
||||
.collect();
|
||||
|
||||
let worker = make_worker(tools).await;
|
||||
|
||||
let selections: Vec<ToolSelection> = (0..3)
|
||||
.map(|i| ToolSelection {
|
||||
tool_name: format!("slow_{}", i),
|
||||
parameters: serde_json::json!({}),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: format!("call_{}", i),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let results = worker.execute_tools_parallel(&selections).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert_eq!(results.len(), 3);
|
||||
for r in &results {
|
||||
assert!(r.result.is_ok(), "Tool should succeed");
|
||||
}
|
||||
// Parallel should complete well under the sequential 600ms threshold.
|
||||
assert!(
|
||||
elapsed < Duration::from_millis(500),
|
||||
"Parallel execution took {:?}, expected < 500ms",
|
||||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_result_ordering_preserved() {
|
||||
// Tools with different delays finish in different order.
|
||||
// Results must be returned in the original request order.
|
||||
let tools: Vec<Arc<dyn Tool>> = vec![
|
||||
Arc::new(SlowTool {
|
||||
tool_name: "tool_a".into(),
|
||||
delay: Duration::from_millis(300),
|
||||
}),
|
||||
Arc::new(SlowTool {
|
||||
tool_name: "tool_b".into(),
|
||||
delay: Duration::from_millis(100),
|
||||
}),
|
||||
Arc::new(SlowTool {
|
||||
tool_name: "tool_c".into(),
|
||||
delay: Duration::from_millis(200),
|
||||
}),
|
||||
];
|
||||
|
||||
let worker = make_worker(tools).await;
|
||||
|
||||
let selections = vec![
|
||||
ToolSelection {
|
||||
tool_name: "tool_a".into(),
|
||||
parameters: serde_json::json!({}),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: "call_a".into(),
|
||||
},
|
||||
ToolSelection {
|
||||
tool_name: "tool_b".into(),
|
||||
parameters: serde_json::json!({}),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: "call_b".into(),
|
||||
},
|
||||
ToolSelection {
|
||||
tool_name: "tool_c".into(),
|
||||
parameters: serde_json::json!({}),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: "call_c".into(),
|
||||
},
|
||||
];
|
||||
|
||||
let results = worker.execute_tools_parallel(&selections).await;
|
||||
|
||||
// Results must be in same order as selections, not completion order.
|
||||
assert!(results[0].result.as_ref().unwrap().contains("done_tool_a"));
|
||||
assert!(results[1].result.as_ref().unwrap().contains("done_tool_b"));
|
||||
assert!(results[2].result.as_ref().unwrap().contains("done_tool_c"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_missing_tool_produces_error_not_panic() {
|
||||
// If a tool doesn't exist, the result slot should contain an error.
|
||||
let worker = make_worker(vec![]).await;
|
||||
|
||||
let selections = vec![ToolSelection {
|
||||
tool_name: "nonexistent_tool".into(),
|
||||
parameters: serde_json::json!({}),
|
||||
reasoning: String::new(),
|
||||
alternatives: vec![],
|
||||
tool_call_id: "call_x".into(),
|
||||
}];
|
||||
|
||||
let results = worker.execute_tools_parallel(&selections).await;
|
||||
assert_eq!(results.len(), 1);
|
||||
assert!(
|
||||
results[0].result.is_err(),
|
||||
"Missing tool should produce an error, not a panic"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+61
-1
@@ -103,7 +103,67 @@ pub fn save_bootstrap_env(vars: &[(&str, &str)]) -> std::io::Result<()> {
|
||||
let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
content.push_str(&format!("{}=\"{}\"\n", key, escaped));
|
||||
}
|
||||
std::fs::write(&path, content)
|
||||
std::fs::write(&path, &content)?;
|
||||
restrict_file_permissions(&path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Update or add a single variable in `~/.ironclaw/.env`, preserving existing content.
|
||||
///
|
||||
/// Unlike `save_bootstrap_env` (which overwrites the entire file), this
|
||||
/// reads the current `.env`, replaces the line for `key` if it exists,
|
||||
/// or appends it otherwise. Use this when writing a single bootstrap var
|
||||
/// outside the wizard (which manages the full set via `save_bootstrap_env`).
|
||||
pub fn upsert_bootstrap_var(key: &str, value: &str) -> std::io::Result<()> {
|
||||
let path = ironclaw_env_path();
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
let escaped = value.replace('\\', "\\\\").replace('"', "\\\"");
|
||||
let new_line = format!("{}=\"{}\"", key, escaped);
|
||||
let prefix = format!("{}=", key);
|
||||
|
||||
let existing = std::fs::read_to_string(&path).unwrap_or_default();
|
||||
|
||||
let mut found = false;
|
||||
let mut result = String::new();
|
||||
for line in existing.lines() {
|
||||
if line.starts_with(&prefix) {
|
||||
if !found {
|
||||
result.push_str(&new_line);
|
||||
result.push('\n');
|
||||
found = true;
|
||||
}
|
||||
// Skip duplicate lines for this key
|
||||
continue;
|
||||
}
|
||||
result.push_str(line);
|
||||
result.push('\n');
|
||||
}
|
||||
|
||||
if !found {
|
||||
result.push_str(&new_line);
|
||||
result.push('\n');
|
||||
}
|
||||
|
||||
std::fs::write(&path, result)?;
|
||||
restrict_file_permissions(&path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set restrictive file permissions (0o600) on Unix systems.
|
||||
///
|
||||
/// The `.env` file may contain database credentials and API keys,
|
||||
/// so it should only be readable by the owner.
|
||||
fn restrict_file_permissions(_path: &std::path::Path) -> std::io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let perms = std::fs::Permissions::from_mode(0o600);
|
||||
std::fs::set_permissions(_path, perms)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write `DATABASE_URL` to `~/.ironclaw/.env`.
|
||||
|
||||
@@ -17,6 +17,7 @@ mod mcp;
|
||||
pub mod memory;
|
||||
pub mod oauth_defaults;
|
||||
mod pairing;
|
||||
mod registry;
|
||||
mod service;
|
||||
pub mod status;
|
||||
mod tool;
|
||||
@@ -29,6 +30,7 @@ pub use memory::MemoryCommand;
|
||||
pub use memory::run_memory_command;
|
||||
pub use memory::run_memory_command_with_db;
|
||||
pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store};
|
||||
pub use registry::{RegistryCommand, run_registry_command};
|
||||
pub use service::{ServiceCommand, run_service_command};
|
||||
pub use status::run_status_command;
|
||||
pub use tool::{ToolCommand, run_tool_command};
|
||||
@@ -90,6 +92,10 @@ pub enum Command {
|
||||
#[command(subcommand)]
|
||||
Tool(ToolCommand),
|
||||
|
||||
/// Browse and install extensions from the registry
|
||||
#[command(subcommand)]
|
||||
Registry(RegistryCommand),
|
||||
|
||||
/// Manage MCP servers (hosted tool providers)
|
||||
#[command(subcommand)]
|
||||
Mcp(McpCommand),
|
||||
|
||||
@@ -62,6 +62,18 @@ pub fn builtin_credentials(secret_name: &str) -> Option<OAuthCredentials> {
|
||||
/// `http://localhost:9876/callback` (or `/auth/callback` for NEAR AI).
|
||||
pub const OAUTH_CALLBACK_PORT: u16 = 9876;
|
||||
|
||||
/// Returns the OAuth callback base URL.
|
||||
///
|
||||
/// Checks `IRONCLAW_OAUTH_CALLBACK_URL` env var first (useful for remote/VPS
|
||||
/// deployments where `127.0.0.1` is unreachable from the user's browser),
|
||||
/// then falls back to `http://127.0.0.1:{OAUTH_CALLBACK_PORT}`.
|
||||
pub fn callback_url() -> String {
|
||||
std::env::var("IRONCLAW_OAUTH_CALLBACK_URL")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| format!("http://127.0.0.1:{}", OAUTH_CALLBACK_PORT))
|
||||
}
|
||||
|
||||
/// Error from the OAuth callback listener.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OAuthCallbackError {
|
||||
@@ -297,7 +309,54 @@ pub fn landing_html(provider_name: &str, success: bool) -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::cli::oauth_defaults::{builtin_credentials, landing_html};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::cli::oauth_defaults::{builtin_credentials, callback_url, landing_html};
|
||||
|
||||
/// Serializes env-mutating tests to prevent parallel races.
|
||||
static ENV_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn test_callback_url_default() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
// Clear the env var to test default behavior
|
||||
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
}
|
||||
let url = callback_url();
|
||||
assert_eq!(url, "http://127.0.0.1:9876");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_url_env_override() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"IRONCLAW_OAUTH_CALLBACK_URL",
|
||||
"https://myserver.example.com:9876",
|
||||
);
|
||||
}
|
||||
let url = callback_url();
|
||||
assert_eq!(url, "https://myserver.example.com:9876");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
} else {
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_provider_returns_none() {
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
//! Registry CLI commands for discovering and installing extensions.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Subcommand;
|
||||
|
||||
use crate::registry::catalog::RegistryCatalog;
|
||||
use crate::registry::installer::RegistryInstaller;
|
||||
use crate::registry::manifest::ManifestKind;
|
||||
|
||||
#[derive(Subcommand, Debug, Clone)]
|
||||
pub enum RegistryCommand {
|
||||
/// List available extensions in the registry
|
||||
List {
|
||||
/// Filter by kind: "tool" or "channel"
|
||||
#[arg(short, long)]
|
||||
kind: Option<String>,
|
||||
|
||||
/// Filter by tag (e.g. "default", "google", "messaging")
|
||||
#[arg(short, long)]
|
||||
tag: Option<String>,
|
||||
|
||||
/// Show detailed information
|
||||
#[arg(short, long)]
|
||||
verbose: bool,
|
||||
},
|
||||
|
||||
/// Show detailed information about an extension or bundle
|
||||
Info {
|
||||
/// Extension or bundle name (e.g. "slack", "google", "tools/gmail")
|
||||
name: String,
|
||||
},
|
||||
|
||||
/// Install an extension or bundle from the registry
|
||||
Install {
|
||||
/// Extension or bundle name (e.g. "slack", "google", "default")
|
||||
name: String,
|
||||
|
||||
/// Force overwrite if already installed
|
||||
#[arg(short, long)]
|
||||
force: bool,
|
||||
|
||||
/// Build from source instead of downloading pre-built artifact
|
||||
#[arg(long)]
|
||||
build: bool,
|
||||
},
|
||||
|
||||
/// Install the default bundle of recommended extensions
|
||||
InstallDefaults {
|
||||
/// Force overwrite if already installed
|
||||
#[arg(short, long)]
|
||||
force: bool,
|
||||
|
||||
/// Build from source instead of downloading pre-built artifact
|
||||
#[arg(long)]
|
||||
build: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// Run a registry command.
|
||||
pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
|
||||
let registry_dir = find_registry_dir()?;
|
||||
let catalog = RegistryCatalog::load(®istry_dir)?;
|
||||
|
||||
match cmd {
|
||||
RegistryCommand::List { kind, tag, verbose } => {
|
||||
cmd_list(&catalog, kind.as_deref(), tag.as_deref(), verbose)
|
||||
}
|
||||
RegistryCommand::Info { name } => cmd_info(&catalog, &name),
|
||||
RegistryCommand::Install { name, force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, &name, force, build).await
|
||||
}
|
||||
RegistryCommand::InstallDefaults { force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, "default", force, build).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the registry directory by looking relative to the current executable or cwd.
|
||||
fn find_registry_dir() -> anyhow::Result<PathBuf> {
|
||||
// Try relative to current directory (for dev usage)
|
||||
let cwd = std::env::current_dir()?;
|
||||
let candidate = cwd.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
// Try relative to executable (covers installed binary, target/debug/, target/release/)
|
||||
if let Ok(exe) = std::env::current_exe()
|
||||
&& let Some(parent) = exe.parent()
|
||||
{
|
||||
// Walk up to 3 levels: exe dir, parent (target/release → target), grandparent (→ repo root)
|
||||
let mut dir = Some(parent);
|
||||
for _ in 0..3 {
|
||||
if let Some(d) = dir {
|
||||
let candidate = d.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
dir = d.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let candidate = manifest_dir.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
anyhow::bail!(
|
||||
"Could not find registry/ directory. Run from the ironclaw repo root, \
|
||||
or ensure registry/ is next to the ironclaw binary."
|
||||
)
|
||||
}
|
||||
|
||||
fn cmd_list(
|
||||
catalog: &RegistryCatalog,
|
||||
kind: Option<&str>,
|
||||
tag: Option<&str>,
|
||||
verbose: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let kind_filter = match kind {
|
||||
Some("tool" | "tools") => Some(ManifestKind::Tool),
|
||||
Some("channel" | "channels") => Some(ManifestKind::Channel),
|
||||
Some(other) => anyhow::bail!("Unknown kind '{}'. Use 'tool' or 'channel'.", other),
|
||||
None => None,
|
||||
};
|
||||
|
||||
let manifests = catalog.list(kind_filter, tag);
|
||||
|
||||
if manifests.is_empty() {
|
||||
println!("No extensions found matching the criteria.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Print header
|
||||
if verbose {
|
||||
println!(
|
||||
"{:<20} {:<8} {:<8} {:<10} DESCRIPTION",
|
||||
"NAME", "KIND", "VERSION", "AUTH"
|
||||
);
|
||||
println!("{}", "-".repeat(80));
|
||||
} else {
|
||||
println!("{:<20} {:<8} DESCRIPTION", "NAME", "KIND");
|
||||
println!("{}", "-".repeat(60));
|
||||
}
|
||||
|
||||
for m in &manifests {
|
||||
if verbose {
|
||||
let auth = m
|
||||
.auth_summary
|
||||
.as_ref()
|
||||
.and_then(|a| a.method.as_deref())
|
||||
.unwrap_or("none");
|
||||
println!(
|
||||
"{:<20} {:<8} {:<8} {:<10} {}",
|
||||
m.name, m.kind, m.version, auth, m.description
|
||||
);
|
||||
} else {
|
||||
println!("{:<20} {:<8} {}", m.name, m.kind, m.description);
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n{} extension(s) found.", manifests.len());
|
||||
|
||||
// Show bundles hint
|
||||
let bundle_names = catalog.bundle_names();
|
||||
if !bundle_names.is_empty() {
|
||||
println!("\nBundles available: {}", bundle_names.join(", "));
|
||||
println!("Use `ironclaw registry info <bundle>` for details.");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmd_info(catalog: &RegistryCatalog, name: &str) -> anyhow::Result<()> {
|
||||
// Check if it's a bundle
|
||||
if let Some(bundle) = catalog.get_bundle(name) {
|
||||
println!("Bundle: {}", bundle.display_name);
|
||||
if let Some(desc) = &bundle.description {
|
||||
println!(" {}", desc);
|
||||
}
|
||||
println!("\nExtensions:");
|
||||
for ext_key in &bundle.extensions {
|
||||
if let Some(m) = catalog.get(ext_key) {
|
||||
println!(" {} - {} ({})", ext_key, m.description, m.kind);
|
||||
} else {
|
||||
println!(" {} (not found in registry)", ext_key);
|
||||
}
|
||||
}
|
||||
if let Some(shared) = &bundle.shared_auth {
|
||||
println!("\nShared auth: {}", shared);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Single extension (use get_strict to surface ambiguous bare names)
|
||||
let manifest = catalog
|
||||
.get_strict(name)
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
println!("{} ({})", manifest.display_name, manifest.kind);
|
||||
println!(" Version: {}", manifest.version);
|
||||
println!(" {}", manifest.description);
|
||||
|
||||
if !manifest.keywords.is_empty() {
|
||||
println!(" Keywords: {}", manifest.keywords.join(", "));
|
||||
}
|
||||
|
||||
println!("\nSource:");
|
||||
println!(" Directory: {}", manifest.source.dir);
|
||||
println!(" Crate: {}", manifest.source.crate_name);
|
||||
println!(" Capabilities: {}", manifest.source.capabilities);
|
||||
|
||||
if let Some(artifact) = manifest.artifacts.get("wasm32-wasip2") {
|
||||
println!("\nArtifact (wasm32-wasip2):");
|
||||
match &artifact.url {
|
||||
Some(url) => println!(" URL: {}", url),
|
||||
None => println!(" URL: (not yet published)"),
|
||||
}
|
||||
match &artifact.sha256 {
|
||||
Some(sha) => println!(" SHA256: {}", sha),
|
||||
None => println!(" SHA256: (not yet computed)"),
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(auth) = &manifest.auth_summary {
|
||||
println!("\nAuthentication:");
|
||||
if let Some(method) = &auth.method {
|
||||
println!(" Method: {}", method);
|
||||
}
|
||||
if let Some(provider) = &auth.provider {
|
||||
println!(" Provider: {}", provider);
|
||||
}
|
||||
if !auth.secrets.is_empty() {
|
||||
println!(" Secrets: {}", auth.secrets.join(", "));
|
||||
}
|
||||
if let Some(shared) = &auth.shared_auth {
|
||||
println!(" Shared with: {}", shared);
|
||||
}
|
||||
if let Some(url) = &auth.setup_url {
|
||||
println!(" Setup: {}", url);
|
||||
}
|
||||
}
|
||||
|
||||
if !manifest.tags.is_empty() {
|
||||
println!("\nTags: {}", manifest.tags.join(", "));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cmd_install(
|
||||
catalog: &RegistryCatalog,
|
||||
registry_dir: &std::path::Path,
|
||||
name: &str,
|
||||
force: bool,
|
||||
prefer_build: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
// Registry dir parent is the repo root
|
||||
let repo_root = registry_dir
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("Cannot determine repo root from registry dir"))?;
|
||||
|
||||
let installer = RegistryInstaller::with_defaults(repo_root.to_path_buf());
|
||||
|
||||
let (manifests, bundle) = catalog.resolve(name)?;
|
||||
|
||||
if manifests.is_empty() {
|
||||
anyhow::bail!("No extensions found for '{}'.", name);
|
||||
}
|
||||
|
||||
if let Some(bundle_def) = bundle {
|
||||
// Bundle install
|
||||
println!(
|
||||
"Installing bundle '{}' ({} extensions)...\n",
|
||||
bundle_def.display_name,
|
||||
manifests.len()
|
||||
);
|
||||
|
||||
let (outcomes, hints) = installer
|
||||
.install_bundle(&manifests, bundle_def, force, prefer_build)
|
||||
.await;
|
||||
|
||||
println!("\n--- Results ---");
|
||||
for outcome in &outcomes {
|
||||
let caps_status = if outcome.has_capabilities { "+" } else { "-" };
|
||||
println!(
|
||||
" [{}] {} ({}) -> {}",
|
||||
caps_status,
|
||||
outcome.name,
|
||||
outcome.kind,
|
||||
outcome.wasm_path.display()
|
||||
);
|
||||
for w in &outcome.warnings {
|
||||
println!(" Warning: {}", w);
|
||||
}
|
||||
}
|
||||
|
||||
if !hints.is_empty() {
|
||||
println!("\nAuth setup:");
|
||||
for hint in &hints {
|
||||
println!("{}", hint);
|
||||
}
|
||||
}
|
||||
|
||||
println!(
|
||||
"\nInstalled {}/{} extensions.",
|
||||
outcomes.len(),
|
||||
manifests.len()
|
||||
);
|
||||
} else {
|
||||
// Single extension
|
||||
let manifest = manifests[0];
|
||||
let outcome = installer.install(manifest, force, prefer_build).await?;
|
||||
|
||||
println!("\nInstalled successfully:");
|
||||
println!(" Name: {}", outcome.name);
|
||||
println!(" Kind: {}", outcome.kind);
|
||||
println!(" WASM: {}", outcome.wasm_path.display());
|
||||
println!(" Capabilities: {}", outcome.has_capabilities);
|
||||
|
||||
if let Some(auth) = &manifest.auth_summary
|
||||
&& auth.method.as_deref() != Some("none")
|
||||
{
|
||||
println!(
|
||||
"\nNext step: authenticate with `ironclaw tool auth {}`",
|
||||
manifest.name
|
||||
);
|
||||
if let Some(url) = &auth.setup_url {
|
||||
println!(" Setup credentials at: {}", url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -102,16 +102,12 @@ impl EmbeddingsConfig {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::helpers::ENV_MUTEX;
|
||||
use crate::settings::{EmbeddingsSettings, Settings};
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Serializes env-mutating tests to prevent parallel races.
|
||||
static ENV_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Clear all embedding-related env vars.
|
||||
fn clear_embedding_env() {
|
||||
// SAFETY: Only called under ENV_MUTEX in tests. No other threads
|
||||
// observe these vars while the lock is held.
|
||||
// SAFETY: Only called under ENV_MUTEX in tests.
|
||||
unsafe {
|
||||
std::env::remove_var("EMBEDDING_ENABLED");
|
||||
std::env::remove_var("EMBEDDING_PROVIDER");
|
||||
|
||||
@@ -2,6 +2,15 @@ use crate::error::ConfigError;
|
||||
|
||||
use super::INJECTED_VARS;
|
||||
|
||||
/// Crate-wide mutex for tests that mutate process environment variables.
|
||||
///
|
||||
/// The process environment is global state shared across all threads.
|
||||
/// Per-module mutexes do NOT prevent races between modules running in
|
||||
/// parallel. Every `unsafe { set_var / remove_var }` call in tests
|
||||
/// MUST hold this single lock.
|
||||
#[cfg(test)]
|
||||
pub(crate) static ENV_MUTEX: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
pub(crate) fn optional_env(key: &str) -> Result<Option<String>, ConfigError> {
|
||||
// Check real env vars first (always win over injected secrets)
|
||||
match std::env::var(key) {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
use crate::config::helpers::optional_env;
|
||||
use crate::error::ConfigError;
|
||||
|
||||
/// Memory hygiene configuration.
|
||||
///
|
||||
/// Controls automatic cleanup of stale workspace documents.
|
||||
/// Maps to `crate::workspace::hygiene::HygieneConfig`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HygieneConfig {
|
||||
/// Whether hygiene is enabled. Env: `MEMORY_HYGIENE_ENABLED` (default: true).
|
||||
pub enabled: bool,
|
||||
/// Days before `daily/` documents are deleted. Env: `MEMORY_HYGIENE_RETENTION_DAYS` (default: 30).
|
||||
pub retention_days: u32,
|
||||
/// Minimum hours between hygiene passes. Env: `MEMORY_HYGIENE_CADENCE_HOURS` (default: 12).
|
||||
pub cadence_hours: u32,
|
||||
}
|
||||
|
||||
impl Default for HygieneConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
retention_days: 30,
|
||||
cadence_hours: 12,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HygieneConfig {
|
||||
pub(crate) fn resolve() -> Result<Self, ConfigError> {
|
||||
Ok(Self {
|
||||
enabled: optional_env("MEMORY_HYGIENE_ENABLED")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "MEMORY_HYGIENE_ENABLED".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(true),
|
||||
retention_days: optional_env("MEMORY_HYGIENE_RETENTION_DAYS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "MEMORY_HYGIENE_RETENTION_DAYS".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?
|
||||
.unwrap_or(30),
|
||||
cadence_hours: optional_env("MEMORY_HYGIENE_CADENCE_HOURS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "MEMORY_HYGIENE_CADENCE_HOURS".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?
|
||||
.unwrap_or(12),
|
||||
})
|
||||
}
|
||||
|
||||
/// Convert to the workspace hygiene config, resolving the state directory
|
||||
/// to the standard `~/.ironclaw` location.
|
||||
pub fn to_workspace_config(&self) -> crate::workspace::hygiene::HygieneConfig {
|
||||
crate::workspace::hygiene::HygieneConfig {
|
||||
enabled: self.enabled,
|
||||
retention_days: self.retention_days,
|
||||
cadence_hours: self.cadence_hours,
|
||||
state_dir: dirs::home_dir()
|
||||
.unwrap_or_else(|| std::path::PathBuf::from("."))
|
||||
.join(".ironclaw"),
|
||||
}
|
||||
}
|
||||
}
|
||||
+19
-12
@@ -122,12 +122,15 @@ pub struct LlmConfig {
|
||||
}
|
||||
|
||||
/// API mode for NEAR AI.
|
||||
///
|
||||
/// - `Responses` = **NEAR AI Chat** (`private.near.ai`, session token auth)
|
||||
/// - `ChatCompletions` = **NEAR AI Cloud** (`cloud-api.near.ai`, API key auth)
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum NearAiApiMode {
|
||||
/// Use the Responses API (chat-api proxy) - session-based auth
|
||||
/// NEAR AI Chat: Responses API with session token auth
|
||||
#[default]
|
||||
Responses,
|
||||
/// Use the Chat Completions API (cloud-api) - API key auth
|
||||
/// NEAR AI Cloud: Chat Completions API with API key auth
|
||||
ChatCompletions,
|
||||
}
|
||||
|
||||
@@ -148,7 +151,7 @@ impl std::str::FromStr for NearAiApiMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// NEAR AI chat-api configuration.
|
||||
/// NEAR AI configuration (shared by Chat and Cloud modes).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NearAiConfig {
|
||||
/// Model to use (e.g., "claude-3-5-sonnet-20241022", "gpt-4o")
|
||||
@@ -156,15 +159,17 @@ pub struct NearAiConfig {
|
||||
/// Cheap/fast model for lightweight tasks (heartbeat, routing, evaluation).
|
||||
/// Falls back to the main model if not set.
|
||||
pub cheap_model: Option<String>,
|
||||
/// Base URL for the NEAR AI API (default: https://private.near.ai).
|
||||
/// Base URL for the NEAR AI API.
|
||||
/// Chat mode default: `https://private.near.ai`
|
||||
/// Cloud mode default: `https://cloud-api.near.ai`
|
||||
pub base_url: String,
|
||||
/// Base URL for auth/refresh endpoints (default: https://private.near.ai)
|
||||
pub auth_base_url: String,
|
||||
/// Path to session file (default: ~/.ironclaw/session.json)
|
||||
pub session_path: PathBuf,
|
||||
/// API mode: "responses" (chat-api) or "chat_completions" (cloud-api)
|
||||
/// API mode: NEAR AI Chat (Responses) or NEAR AI Cloud (ChatCompletions)
|
||||
pub api_mode: NearAiApiMode,
|
||||
/// API key for cloud-api (required for chat_completions mode)
|
||||
/// API key for NEAR AI Cloud (required for ChatCompletions mode)
|
||||
pub api_key: Option<SecretString>,
|
||||
/// Optional fallback model for failover (default: None).
|
||||
/// When set, a secondary provider is created with this model and wrapped
|
||||
@@ -243,8 +248,13 @@ impl LlmConfig {
|
||||
.to_string()
|
||||
}),
|
||||
cheap_model: optional_env("NEARAI_CHEAP_MODEL")?,
|
||||
base_url: optional_env("NEARAI_BASE_URL")?
|
||||
.unwrap_or_else(|| "https://private.near.ai".to_string()),
|
||||
base_url: optional_env("NEARAI_BASE_URL")?.unwrap_or_else(|| {
|
||||
if api_mode == NearAiApiMode::ChatCompletions {
|
||||
"https://cloud-api.near.ai".to_string()
|
||||
} else {
|
||||
"https://private.near.ai".to_string()
|
||||
}
|
||||
}),
|
||||
auth_base_url: optional_env("NEARAI_AUTH_URL")?
|
||||
.unwrap_or_else(|| "https://private.near.ai".to_string()),
|
||||
session_path: optional_env("NEARAI_SESSION_PATH")?
|
||||
@@ -373,11 +383,8 @@ fn default_session_path() -> PathBuf {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::helpers::ENV_MUTEX;
|
||||
use crate::settings::Settings;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Serializes env-mutating tests to prevent parallel races.
|
||||
static ENV_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// Clear all openai-compatible-related env vars.
|
||||
fn clear_openai_compatible_env() {
|
||||
|
||||
@@ -12,6 +12,7 @@ mod database;
|
||||
mod embeddings;
|
||||
mod heartbeat;
|
||||
pub(crate) mod helpers;
|
||||
mod hygiene;
|
||||
mod llm;
|
||||
mod routines;
|
||||
mod safety;
|
||||
@@ -34,6 +35,7 @@ pub use self::channels::{ChannelsConfig, CliConfig, GatewayConfig, HttpConfig};
|
||||
pub use self::database::{DatabaseBackend, DatabaseConfig, default_libsql_path};
|
||||
pub use self::embeddings::EmbeddingsConfig;
|
||||
pub use self::heartbeat::HeartbeatConfig;
|
||||
pub use self::hygiene::HygieneConfig;
|
||||
pub use self::llm::{
|
||||
AnthropicDirectConfig, LlmBackend, LlmConfig, NearAiApiMode, NearAiConfig, OllamaConfig,
|
||||
OpenAiCompatibleConfig, OpenAiDirectConfig, TinfoilConfig,
|
||||
@@ -67,6 +69,7 @@ pub struct Config {
|
||||
pub secrets: SecretsConfig,
|
||||
pub builder: BuilderModeConfig,
|
||||
pub heartbeat: HeartbeatConfig,
|
||||
pub hygiene: HygieneConfig,
|
||||
pub routines: RoutineConfig,
|
||||
pub sandbox: SandboxModeConfig,
|
||||
pub claude_code: ClaudeCodeConfig,
|
||||
@@ -190,6 +193,7 @@ impl Config {
|
||||
secrets: SecretsConfig::resolve().await?,
|
||||
builder: BuilderModeConfig::resolve()?,
|
||||
heartbeat: HeartbeatConfig::resolve(settings)?,
|
||||
hygiene: HygieneConfig::resolve()?,
|
||||
routines: RoutineConfig::resolve()?,
|
||||
sandbox: SandboxModeConfig::resolve()?,
|
||||
claude_code: ClaudeCodeConfig::resolve()?,
|
||||
@@ -215,6 +219,7 @@ pub async fn inject_llm_keys_from_secrets(
|
||||
("llm_openai_api_key", "OPENAI_API_KEY"),
|
||||
("llm_anthropic_api_key", "ANTHROPIC_API_KEY"),
|
||||
("llm_compatible_api_key", "LLM_API_KEY"),
|
||||
("llm_nearai_api_key", "NEARAI_API_KEY"),
|
||||
];
|
||||
|
||||
let mut injected = HashMap::new();
|
||||
|
||||
@@ -320,10 +320,10 @@ pub(crate) fn row_to_routine_libsql(row: &libsql::Row) -> Result<Routine, Databa
|
||||
let max_concurrent = get_i64(row, 10);
|
||||
let dedup_window_secs: Option<i64> = row.get::<i64>(11).ok();
|
||||
|
||||
let trigger =
|
||||
Trigger::from_db(&trigger_type, trigger_config).map_err(DatabaseError::Serialization)?;
|
||||
let trigger = Trigger::from_db(&trigger_type, trigger_config)
|
||||
.map_err(|e| DatabaseError::Serialization(e.to_string()))?;
|
||||
let action = RoutineAction::from_db(&action_type, action_config)
|
||||
.map_err(DatabaseError::Serialization)?;
|
||||
.map_err(|e| DatabaseError::Serialization(e.to_string()))?;
|
||||
|
||||
Ok(Routine {
|
||||
id: get_text(row, 0).parse().unwrap_or_default(),
|
||||
@@ -359,7 +359,7 @@ pub(crate) fn row_to_routine_run_libsql(row: &libsql::Row) -> Result<RoutineRun,
|
||||
let status_str = get_text(row, 5);
|
||||
let status: RunStatus = status_str
|
||||
.parse()
|
||||
.map_err(|e: String| DatabaseError::Serialization(e))?;
|
||||
.map_err(|e: crate::error::RoutineError| DatabaseError::Serialization(e.to_string()))?;
|
||||
|
||||
Ok(RoutineRun {
|
||||
id: get_text(row, 0).parse().unwrap_or_default(),
|
||||
|
||||
@@ -48,6 +48,9 @@ pub enum Error {
|
||||
|
||||
#[error("Worker error: {0}")]
|
||||
Worker(#[from] WorkerError),
|
||||
|
||||
#[error("Routine error: {0}")]
|
||||
Routine(#[from] RoutineError),
|
||||
}
|
||||
|
||||
/// Configuration-related errors.
|
||||
@@ -365,5 +368,45 @@ pub enum WorkerError {
|
||||
MissingToken,
|
||||
}
|
||||
|
||||
/// Routine-related errors.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RoutineError {
|
||||
#[error("Unknown trigger type: {trigger_type}")]
|
||||
UnknownTriggerType { trigger_type: String },
|
||||
|
||||
#[error("Unknown action type: {action_type}")]
|
||||
UnknownActionType { action_type: String },
|
||||
|
||||
#[error("Missing field in {context}: {field}")]
|
||||
MissingField { context: String, field: String },
|
||||
|
||||
#[error("Invalid cron expression: {reason}")]
|
||||
InvalidCron { reason: String },
|
||||
|
||||
#[error("Unknown run status: {status}")]
|
||||
UnknownRunStatus { status: String },
|
||||
|
||||
#[error("Routine {name} is disabled")]
|
||||
Disabled { name: String },
|
||||
|
||||
#[error("Routine not found: {id}")]
|
||||
NotFound { id: Uuid },
|
||||
|
||||
#[error("Routine {name} at max concurrent runs")]
|
||||
MaxConcurrent { name: String },
|
||||
|
||||
#[error("Database error: {reason}")]
|
||||
Database { reason: String },
|
||||
|
||||
#[error("LLM call failed: {reason}")]
|
||||
LlmFailed { reason: String },
|
||||
|
||||
#[error("LLM returned empty content")]
|
||||
EmptyResponse,
|
||||
|
||||
#[error("LLM response truncated (finish_reason=length) with no content")]
|
||||
TruncatedResponse,
|
||||
}
|
||||
|
||||
/// Result type alias for the agent.
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
@@ -1179,10 +1179,10 @@ fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
||||
let max_concurrent: i32 = row.get("max_concurrent");
|
||||
let dedup_window_secs: Option<i32> = row.get("dedup_window_secs");
|
||||
|
||||
let trigger =
|
||||
Trigger::from_db(&trigger_type, trigger_config).map_err(DatabaseError::Serialization)?;
|
||||
let trigger = Trigger::from_db(&trigger_type, trigger_config)
|
||||
.map_err(|e| DatabaseError::Serialization(e.to_string()))?;
|
||||
let action = RoutineAction::from_db(&action_type, action_config)
|
||||
.map_err(DatabaseError::Serialization)?;
|
||||
.map_err(|e| DatabaseError::Serialization(e.to_string()))?;
|
||||
|
||||
Ok(Routine {
|
||||
id: row.get("id"),
|
||||
@@ -1219,7 +1219,7 @@ fn row_to_routine_run(row: &tokio_postgres::Row) -> Result<RoutineRun, DatabaseE
|
||||
let status_str: String = row.get("status");
|
||||
let status: RunStatus = status_str
|
||||
.parse()
|
||||
.map_err(|e: String| DatabaseError::Serialization(e))?;
|
||||
.map_err(|e: crate::error::RoutineError| DatabaseError::Serialization(e.to_string()))?;
|
||||
|
||||
Ok(RoutineRun {
|
||||
id: row.get("id"),
|
||||
|
||||
@@ -57,6 +57,7 @@ pub mod llm;
|
||||
pub mod observability;
|
||||
pub mod orchestrator;
|
||||
pub mod pairing;
|
||||
pub mod registry;
|
||||
pub mod safety;
|
||||
pub mod sandbox;
|
||||
pub mod secrets;
|
||||
|
||||
+38
-12
@@ -17,7 +17,17 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> {
|
||||
.unwrap_or(model_id);
|
||||
|
||||
match id {
|
||||
// OpenAI models -- prices per token (USD)
|
||||
// OpenAI — GPT-5.x / Codex
|
||||
"gpt-5.3-codex" | "gpt-5.3-codex-spark" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"gpt-5.2-codex" | "gpt-5.2-pro" | "gpt-5.2" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"gpt-5.1-codex" | "gpt-5.1-codex-max" | "gpt-5.1" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"gpt-5.1-codex-mini" => Some((dec!(0.0000003), dec!(0.0000012))),
|
||||
"gpt-5-codex" | "gpt-5-pro" | "gpt-5" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"gpt-5-mini" | "gpt-5-nano" => Some((dec!(0.0000003), dec!(0.0000012))),
|
||||
// OpenAI — GPT-4.x
|
||||
"gpt-4.1" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"gpt-4.1-mini" => Some((dec!(0.0000004), dec!(0.0000016))),
|
||||
"gpt-4.1-nano" => Some((dec!(0.0000001), dec!(0.0000004))),
|
||||
"gpt-4o" | "gpt-4o-2024-11-20" | "gpt-4o-2024-08-06" => {
|
||||
Some((dec!(0.0000025), dec!(0.00001)))
|
||||
}
|
||||
@@ -25,20 +35,36 @@ pub fn model_cost(model_id: &str) -> Option<(Decimal, Decimal)> {
|
||||
"gpt-4-turbo" | "gpt-4-turbo-2024-04-09" => Some((dec!(0.00001), dec!(0.00003))),
|
||||
"gpt-4" | "gpt-4-0613" => Some((dec!(0.00003), dec!(0.00006))),
|
||||
"gpt-3.5-turbo" | "gpt-3.5-turbo-0125" => Some((dec!(0.0000005), dec!(0.0000015))),
|
||||
// OpenAI — reasoning
|
||||
"o3" => Some((dec!(0.000002), dec!(0.000008))),
|
||||
"o3-mini" | "o3-mini-2025-01-31" => Some((dec!(0.0000011), dec!(0.0000044))),
|
||||
"o4-mini" => Some((dec!(0.0000011), dec!(0.0000044))),
|
||||
"o1" | "o1-2024-12-17" => Some((dec!(0.000015), dec!(0.00006))),
|
||||
"o1-mini" | "o1-mini-2024-09-12" => Some((dec!(0.000003), dec!(0.000012))),
|
||||
"o3-mini" | "o3-mini-2025-01-31" => Some((dec!(0.0000011), dec!(0.0000044))),
|
||||
|
||||
// Anthropic models
|
||||
"claude-3-5-sonnet-20241022" | "claude-3-5-sonnet-latest" | "claude-sonnet-4-20250514" => {
|
||||
Some((dec!(0.000003), dec!(0.000015)))
|
||||
}
|
||||
"claude-3-5-haiku-20241022" | "claude-3-5-haiku-latest" => {
|
||||
Some((dec!(0.0000008), dec!(0.000004)))
|
||||
}
|
||||
"claude-3-opus-20240229" | "claude-3-opus-latest" | "claude-opus-4-20250514" => {
|
||||
Some((dec!(0.000015), dec!(0.000075)))
|
||||
}
|
||||
// Anthropic
|
||||
"claude-opus-4-6"
|
||||
| "claude-opus-4-5"
|
||||
| "claude-opus-4-5-20251101"
|
||||
| "claude-opus-4-1"
|
||||
| "claude-opus-4-1-20250805"
|
||||
| "claude-opus-4-0"
|
||||
| "claude-opus-4-20250514"
|
||||
| "claude-3-opus-20240229"
|
||||
| "claude-3-opus-latest" => Some((dec!(0.000015), dec!(0.000075))),
|
||||
"claude-sonnet-4-6"
|
||||
| "claude-sonnet-4-5"
|
||||
| "claude-sonnet-4-5-20250929"
|
||||
| "claude-sonnet-4-0"
|
||||
| "claude-sonnet-4-20250514"
|
||||
| "claude-3-7-sonnet-20250219"
|
||||
| "claude-3-7-sonnet-latest"
|
||||
| "claude-3-5-sonnet-20241022"
|
||||
| "claude-3-5-sonnet-latest" => Some((dec!(0.000003), dec!(0.000015))),
|
||||
"claude-haiku-4-5"
|
||||
| "claude-haiku-4-5-20251001"
|
||||
| "claude-3-5-haiku-20241022"
|
||||
| "claude-3-5-haiku-latest" => Some((dec!(0.0000008), dec!(0.000004))),
|
||||
"claude-3-haiku-20240307" => Some((dec!(0.00000025), dec!(0.00000125))),
|
||||
|
||||
// Ollama / local models -- free
|
||||
|
||||
+4
-2
@@ -75,14 +75,16 @@ pub fn create_llm_provider_with_config(
|
||||
NearAiApiMode::Responses => {
|
||||
tracing::info!(
|
||||
model = %config.model,
|
||||
"Using Responses API (chat-api) with session auth"
|
||||
base_url = %config.base_url,
|
||||
"Using NEAR AI Chat (Responses API, session token auth)"
|
||||
);
|
||||
Ok(Arc::new(NearAiProvider::new(config.clone(), session)?))
|
||||
}
|
||||
NearAiApiMode::ChatCompletions => {
|
||||
tracing::info!(
|
||||
model = %config.model,
|
||||
"Using Chat Completions API (cloud-api) with API key auth"
|
||||
base_url = %config.base_url,
|
||||
"Using NEAR AI Cloud (Chat Completions API, API key auth)"
|
||||
);
|
||||
Ok(Arc::new(NearAiChatProvider::new(config.clone())?))
|
||||
}
|
||||
|
||||
+5
-3
@@ -1,7 +1,9 @@
|
||||
//! NEAR AI Chat API provider implementation.
|
||||
//! NEAR AI Chat provider implementation (Responses API).
|
||||
//!
|
||||
//! This provider uses the NEAR AI chat-api which provides a unified interface
|
||||
//! to multiple LLM models (OpenAI, Anthropic, etc.) with user authentication.
|
||||
//! This provider uses the NEAR AI Responses API (`private.near.ai`) which
|
||||
//! provides a unified interface to multiple LLM models with session token
|
||||
//! authentication. Supports response chaining for efficient multi-turn
|
||||
//! conversations.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
//! NEAR AI Chat Completions API provider implementation.
|
||||
//! NEAR AI Cloud provider implementation (Chat Completions API).
|
||||
//!
|
||||
//! This provider uses the standard OpenAI-compatible chat completions API
|
||||
//! with API key authentication (for cloud-api).
|
||||
//! This provider uses the NEAR AI Cloud API (`cloud-api.near.ai`) which
|
||||
//! exposes an OpenAI-compatible chat completions endpoint with API key
|
||||
//! authentication.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use reqwest::Client;
|
||||
@@ -17,7 +18,7 @@ use crate::llm::provider::{
|
||||
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse,
|
||||
};
|
||||
|
||||
/// NEAR AI Chat Completions API provider.
|
||||
/// NEAR AI Cloud provider (Chat Completions API, API key auth).
|
||||
pub struct NearAiChatProvider {
|
||||
client: Client,
|
||||
config: NearAiConfig,
|
||||
@@ -26,10 +27,10 @@ pub struct NearAiChatProvider {
|
||||
}
|
||||
|
||||
impl NearAiChatProvider {
|
||||
/// Create a new NEAR AI chat completions provider with API key auth.
|
||||
/// Create a new NEAR AI Cloud provider with API key auth.
|
||||
///
|
||||
/// By default this enables tool-message flattening for compatibility with
|
||||
/// providers that reject `role: "tool"` messages (e.g. NEAR cloud-api).
|
||||
/// providers that reject `role: "tool"` messages.
|
||||
pub fn new(config: NearAiConfig) -> Result<Self, LlmError> {
|
||||
Self::new_with_flatten(config, true)
|
||||
}
|
||||
|
||||
+120
-43
@@ -217,38 +217,43 @@ impl SessionManager {
|
||||
self.initiate_login().await
|
||||
}
|
||||
|
||||
/// Start the OAuth login flow.
|
||||
/// Start the login flow.
|
||||
///
|
||||
/// 1. Bind the fixed callback port
|
||||
/// Shows the auth method menu FIRST (before binding any listener), so
|
||||
/// that the API-key path can skip network binding entirely. This is
|
||||
/// important for remote/headless servers where `127.0.0.1` is
|
||||
/// unreachable from the user's browser.
|
||||
///
|
||||
/// For OAuth paths (GitHub, Google):
|
||||
/// 1. Bind the callback listener
|
||||
/// 2. Print the auth URL and attempt to open browser
|
||||
/// 3. Wait for OAuth callback with session token
|
||||
/// 4. Save and return the token
|
||||
///
|
||||
/// For NEAR AI Cloud API key:
|
||||
/// 1. Prompt user for API key from cloud.near.ai
|
||||
/// 2. Set NEARAI_API_KEY env var and save to bootstrap .env
|
||||
/// 3. No session token saved (different auth model)
|
||||
async fn initiate_login(&self) -> Result<(), LlmError> {
|
||||
use crate::cli::oauth_defaults::{self, OAUTH_CALLBACK_PORT};
|
||||
use crate::cli::oauth_defaults;
|
||||
|
||||
let listener = oauth_defaults::bind_callback_listener()
|
||||
.await
|
||||
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
let cb_url = oauth_defaults::callback_url();
|
||||
|
||||
let callback_url = format!("http://127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||
|
||||
// Show auth provider menu
|
||||
// Show auth provider menu BEFORE binding the listener
|
||||
println!();
|
||||
println!("╔════════════════════════════════════════════════════════════════╗");
|
||||
println!("║ NEAR AI Authentication ║");
|
||||
println!("╠════════════════════════════════════════════════════════════════╣");
|
||||
println!("║ Choose an authentication method: ║");
|
||||
println!("║ ║");
|
||||
println!("║ [1] GitHub ║");
|
||||
println!("║ [2] Google ║");
|
||||
println!("║ [1] GitHub (requires localhost browser access) ║");
|
||||
println!("║ [2] Google (requires localhost browser access) ║");
|
||||
println!("║ [3] NEAR Wallet (coming soon) ║");
|
||||
println!("║ [4] NEAR AI Cloud API key ║");
|
||||
println!("║ ║");
|
||||
println!("╚════════════════════════════════════════════════════════════════╝");
|
||||
println!();
|
||||
print!("Enter choice [1-3]: ");
|
||||
print!("Enter choice [1-4]: ");
|
||||
|
||||
// Flush stdout to ensure prompt is displayed
|
||||
use std::io::Write;
|
||||
@@ -263,23 +268,8 @@ impl SessionManager {
|
||||
reason: format!("Failed to read input: {}", e),
|
||||
})?;
|
||||
|
||||
let (auth_provider, auth_url) = match choice.trim() {
|
||||
"1" | "" => {
|
||||
let url = format!(
|
||||
"{}/v1/auth/github?frontend_callback={}",
|
||||
self.config.auth_base_url,
|
||||
urlencoding::encode(&callback_url)
|
||||
);
|
||||
("github", url)
|
||||
}
|
||||
"2" => {
|
||||
let url = format!(
|
||||
"{}/v1/auth/google?frontend_callback={}",
|
||||
self.config.auth_base_url,
|
||||
urlencoding::encode(&callback_url)
|
||||
);
|
||||
("google", url)
|
||||
}
|
||||
match choice.trim() {
|
||||
"4" => return self.api_key_login().await,
|
||||
"3" => {
|
||||
println!();
|
||||
println!("NEAR Wallet authentication is not yet implemented.");
|
||||
@@ -289,12 +279,41 @@ impl SessionManager {
|
||||
reason: "NEAR Wallet auth not yet implemented".to_string(),
|
||||
});
|
||||
}
|
||||
_ => {
|
||||
"1" | "" | "2" => {} // handled below after listener bind
|
||||
other => {
|
||||
return Err(LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: format!("Invalid choice: {}", choice.trim()),
|
||||
reason: format!("Invalid choice: {}", other),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// OAuth paths: bind the callback listener now
|
||||
let listener = oauth_defaults::bind_callback_listener()
|
||||
.await
|
||||
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let (auth_provider, auth_url) = match choice.trim() {
|
||||
"2" => {
|
||||
let url = format!(
|
||||
"{}/v1/auth/google?frontend_callback={}",
|
||||
self.config.auth_base_url,
|
||||
urlencoding::encode(&cb_url)
|
||||
);
|
||||
("google", url)
|
||||
}
|
||||
_ => {
|
||||
// "1" or "" (default)
|
||||
let url = format!(
|
||||
"{}/v1/auth/github?frontend_callback={}",
|
||||
self.config.auth_base_url,
|
||||
urlencoding::encode(&cb_url)
|
||||
);
|
||||
("github", url)
|
||||
}
|
||||
};
|
||||
|
||||
println!();
|
||||
@@ -341,6 +360,63 @@ impl SessionManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// NEAR AI Cloud API key entry flow.
|
||||
///
|
||||
/// Prompts the user to enter a NEAR AI Cloud API key from
|
||||
/// cloud.near.ai. The key is set as `NEARAI_API_KEY` env var so
|
||||
/// `LlmConfig::resolve()` auto-selects ChatCompletions mode, and
|
||||
/// saved to `~/.ironclaw/.env` for persistence across restarts.
|
||||
/// No session token is saved and no `/v1/users/me` validation is
|
||||
/// performed (different auth model).
|
||||
async fn api_key_login(&self) -> Result<(), LlmError> {
|
||||
println!();
|
||||
println!("NEAR AI Cloud API key");
|
||||
println!("─────────────────────");
|
||||
println!();
|
||||
println!(" 1. Open https://cloud.near.ai in your browser");
|
||||
println!(" 2. Sign in and navigate to API Keys");
|
||||
println!(" 3. Create or copy an existing API key");
|
||||
println!();
|
||||
|
||||
let key_secret =
|
||||
crate::setup::secret_input("API key").map_err(|e| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: format!("Failed to read input: {}", e),
|
||||
})?;
|
||||
|
||||
use secrecy::ExposeSecret;
|
||||
let key = key_secret.expose_secret().to_string();
|
||||
if key.is_empty() {
|
||||
return Err(LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: "API key cannot be empty".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Set env var so Config picks it up immediately
|
||||
// (LlmConfig::resolve() auto-selects ChatCompletions mode when
|
||||
// NEARAI_API_KEY is present).
|
||||
//
|
||||
// SAFETY: called during single-threaded interactive login flow.
|
||||
#[allow(unused_unsafe)]
|
||||
unsafe {
|
||||
std::env::set_var("NEARAI_API_KEY", &key);
|
||||
}
|
||||
|
||||
// Persist to ~/.ironclaw/.env so the key survives restarts
|
||||
// (bootstrap layer — available before DB is connected).
|
||||
// Uses upsert to avoid clobbering existing bootstrap vars.
|
||||
if let Err(e) = crate::bootstrap::upsert_bootstrap_var("NEARAI_API_KEY", &key) {
|
||||
tracing::warn!("Failed to save API key to bootstrap .env: {}", e);
|
||||
}
|
||||
|
||||
println!();
|
||||
crate::setup::print_success("NEAR AI Cloud API key saved.");
|
||||
println!();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save session data to disk and (if available) to the database.
|
||||
async fn save_session(&self, token: &str, auth_provider: Option<&str>) -> Result<(), LlmError> {
|
||||
let session = SessionData {
|
||||
@@ -508,20 +584,21 @@ impl SessionManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a session manager from a config, migrating from env var if present.
|
||||
/// Create a session manager from a config, loading env var if present.
|
||||
///
|
||||
/// When `NEARAI_SESSION_TOKEN` is set, it takes precedence over file-based
|
||||
/// tokens. This supports hosting providers that inject the token via env var.
|
||||
pub async fn create_session_manager(config: SessionConfig) -> Arc<SessionManager> {
|
||||
let manager = SessionManager::new_async(config).await;
|
||||
|
||||
// Check for legacy env var and migrate if present and no file token
|
||||
if !manager.has_token().await
|
||||
&& let Ok(token) = std::env::var("NEARAI_SESSION_TOKEN")
|
||||
// NEARAI_SESSION_TOKEN env var always takes precedence over file-based
|
||||
// tokens. Hosting providers set this env var and expect it to be used
|
||||
// directly — no file persistence needed.
|
||||
if let Ok(token) = std::env::var("NEARAI_SESSION_TOKEN")
|
||||
&& !token.is_empty()
|
||||
{
|
||||
tracing::info!("Migrating session token from NEARAI_SESSION_TOKEN env var to file");
|
||||
manager.set_token(SecretString::from(token.clone())).await;
|
||||
if let Err(e) = manager.save_session(&token, None).await {
|
||||
tracing::warn!("Failed to save migrated session: {}", e);
|
||||
}
|
||||
tracing::info!("Using session token from NEARAI_SESSION_TOKEN env var");
|
||||
manager.set_token(SecretString::from(token)).await;
|
||||
}
|
||||
|
||||
Arc::new(manager)
|
||||
|
||||
+10
@@ -80,6 +80,15 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
return ironclaw::cli::run_config_command(config_cmd.clone()).await;
|
||||
}
|
||||
Some(Command::Registry(registry_cmd)) => {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")),
|
||||
)
|
||||
.init();
|
||||
|
||||
return ironclaw::cli::run_registry_command(registry_cmd.clone()).await;
|
||||
}
|
||||
Some(Command::Mcp(mcp_cmd)) => {
|
||||
// Simple logging for MCP commands
|
||||
tracing_subscriber::fmt()
|
||||
@@ -1496,6 +1505,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
deps,
|
||||
channels,
|
||||
Some(config.heartbeat.clone()),
|
||||
Some(config.hygiene.clone()),
|
||||
Some(config.routines.clone()),
|
||||
Some(context_manager),
|
||||
Some(session_manager),
|
||||
|
||||
@@ -0,0 +1,580 @@
|
||||
//! Registry catalog: loads manifests from disk, provides list/search/resolve operations.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest, ManifestKind};
|
||||
|
||||
/// Error type for registry operations.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum RegistryError {
|
||||
#[error("Registry directory not found: {0}")]
|
||||
DirectoryNotFound(PathBuf),
|
||||
|
||||
#[error("Failed to read manifest {path}: {reason}")]
|
||||
ManifestRead { path: PathBuf, reason: String },
|
||||
|
||||
#[error("Failed to parse manifest {path}: {reason}")]
|
||||
ManifestParse { path: PathBuf, reason: String },
|
||||
|
||||
#[error("Extension not found: {0}")]
|
||||
ExtensionNotFound(String),
|
||||
|
||||
#[error("'{name}' already installed at {path}. Use --force to overwrite.")]
|
||||
AlreadyInstalled {
|
||||
name: String,
|
||||
path: std::path::PathBuf,
|
||||
},
|
||||
|
||||
#[error("Download failed for {url}: {reason}")]
|
||||
DownloadFailed { url: String, reason: String },
|
||||
|
||||
#[error(
|
||||
"Ambiguous name '{name}': exists as both {kind_a} and {kind_b}. Use '{prefix_a}/{name}' or '{prefix_b}/{name}'."
|
||||
)]
|
||||
AmbiguousName {
|
||||
name: String,
|
||||
kind_a: &'static str,
|
||||
prefix_a: &'static str,
|
||||
kind_b: &'static str,
|
||||
prefix_b: &'static str,
|
||||
},
|
||||
|
||||
#[error("Bundle not found: {0}")]
|
||||
BundleNotFound(String),
|
||||
|
||||
#[error("Failed to read bundles file: {0}")]
|
||||
BundlesRead(String),
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
/// Central catalog loaded from the `registry/` directory.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RegistryCatalog {
|
||||
/// All loaded manifests, keyed by "<kind>/<name>" (e.g. "tools/slack").
|
||||
manifests: HashMap<String, ExtensionManifest>,
|
||||
|
||||
/// Bundle definitions from `_bundles.json`.
|
||||
bundles: HashMap<String, BundleDefinition>,
|
||||
|
||||
/// Root directory of the registry.
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
impl RegistryCatalog {
|
||||
/// Load the catalog from a registry directory.
|
||||
///
|
||||
/// Expects the structure:
|
||||
/// ```text
|
||||
/// registry/
|
||||
/// ├── tools/*.json
|
||||
/// ├── channels/*.json
|
||||
/// └── _bundles.json
|
||||
/// ```
|
||||
pub fn load(registry_dir: &Path) -> Result<Self, RegistryError> {
|
||||
if !registry_dir.exists() {
|
||||
return Err(RegistryError::DirectoryNotFound(registry_dir.to_path_buf()));
|
||||
}
|
||||
|
||||
let mut manifests = HashMap::new();
|
||||
|
||||
// Load tools
|
||||
let tools_dir = registry_dir.join("tools");
|
||||
if tools_dir.is_dir() {
|
||||
Self::load_manifests_from_dir(&tools_dir, "tools", &mut manifests)?;
|
||||
}
|
||||
|
||||
// Load channels
|
||||
let channels_dir = registry_dir.join("channels");
|
||||
if channels_dir.is_dir() {
|
||||
Self::load_manifests_from_dir(&channels_dir, "channels", &mut manifests)?;
|
||||
}
|
||||
|
||||
// Load bundles
|
||||
let bundles_path = registry_dir.join("_bundles.json");
|
||||
let bundles = if bundles_path.is_file() {
|
||||
let content = std::fs::read_to_string(&bundles_path).map_err(|e| {
|
||||
RegistryError::BundlesRead(format!("{}: {}", bundles_path.display(), e))
|
||||
})?;
|
||||
let bundles_file: BundlesFile = serde_json::from_str(&content).map_err(|e| {
|
||||
RegistryError::BundlesRead(format!("{}: {}", bundles_path.display(), e))
|
||||
})?;
|
||||
bundles_file.bundles
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
|
||||
Ok(Self {
|
||||
manifests,
|
||||
bundles,
|
||||
root: registry_dir.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
fn load_manifests_from_dir(
|
||||
dir: &Path,
|
||||
kind_prefix: &str,
|
||||
manifests: &mut HashMap<String, ExtensionManifest>,
|
||||
) -> Result<(), RegistryError> {
|
||||
let entries = std::fs::read_dir(dir).map_err(|e| RegistryError::ManifestRead {
|
||||
path: dir.to_path_buf(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
for entry in entries {
|
||||
let entry = entry.map_err(|e| RegistryError::ManifestRead {
|
||||
path: dir.to_path_buf(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let path = entry.path();
|
||||
if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
|
||||
let content =
|
||||
std::fs::read_to_string(&path).map_err(|e| RegistryError::ManifestRead {
|
||||
path: path.clone(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let manifest: ExtensionManifest =
|
||||
serde_json::from_str(&content).map_err(|e| RegistryError::ManifestParse {
|
||||
path: path.clone(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let key = format!("{}/{}", kind_prefix, manifest.name);
|
||||
manifests.insert(key, manifest);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The root directory this catalog was loaded from.
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
/// Get all manifests.
|
||||
pub fn all(&self) -> Vec<&ExtensionManifest> {
|
||||
let mut items: Vec<_> = self.manifests.values().collect();
|
||||
items.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
items
|
||||
}
|
||||
|
||||
/// List manifests, optionally filtered by kind and/or tag.
|
||||
pub fn list(&self, kind: Option<ManifestKind>, tag: Option<&str>) -> Vec<&ExtensionManifest> {
|
||||
let mut results: Vec<_> = self
|
||||
.manifests
|
||||
.values()
|
||||
.filter(|m| kind.is_none_or(|k| m.kind == k))
|
||||
.filter(|m| tag.is_none_or(|t| m.tags.iter().any(|mt| mt == t)))
|
||||
.collect();
|
||||
results.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
results
|
||||
}
|
||||
|
||||
/// Get a manifest by name. Tries exact key match first ("tools/slack"),
|
||||
/// then searches by bare name ("slack").
|
||||
///
|
||||
/// If a bare name matches both a tool and a channel, returns `None`.
|
||||
/// Use a qualified key ("tools/slack" or "channels/slack") to disambiguate.
|
||||
pub fn get(&self, name: &str) -> Option<&ExtensionManifest> {
|
||||
// Try exact key first
|
||||
if let Some(m) = self.manifests.get(name) {
|
||||
return Some(m);
|
||||
}
|
||||
|
||||
// Try with kind prefix, detecting collisions
|
||||
let tool = self.manifests.get(&format!("tools/{}", name));
|
||||
let channel = self.manifests.get(&format!("channels/{}", name));
|
||||
|
||||
match (tool, channel) {
|
||||
(Some(_), Some(_)) => None, // ambiguous
|
||||
(Some(m), None) => Some(m),
|
||||
(None, Some(m)) => Some(m),
|
||||
(None, None) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a manifest by name, returning a `Result` with an explicit error for
|
||||
/// ambiguous bare names.
|
||||
pub fn get_strict(&self, name: &str) -> Result<&ExtensionManifest, RegistryError> {
|
||||
// Try exact key first
|
||||
if let Some(m) = self.manifests.get(name) {
|
||||
return Ok(m);
|
||||
}
|
||||
|
||||
let has_tool = self.manifests.contains_key(&format!("tools/{}", name));
|
||||
let has_channel = self.manifests.contains_key(&format!("channels/{}", name));
|
||||
|
||||
match (has_tool, has_channel) {
|
||||
(true, true) => Err(RegistryError::AmbiguousName {
|
||||
name: name.to_string(),
|
||||
kind_a: "tool",
|
||||
prefix_a: "tools",
|
||||
kind_b: "channel",
|
||||
prefix_b: "channels",
|
||||
}),
|
||||
(true, false) => Ok(self.manifests.get(&format!("tools/{}", name)).unwrap()),
|
||||
(false, true) => Ok(self.manifests.get(&format!("channels/{}", name)).unwrap()),
|
||||
(false, false) => Err(RegistryError::ExtensionNotFound(name.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the full key ("tools/slack" or "channels/telegram") for a manifest.
|
||||
pub fn key_for(&self, name: &str) -> Option<String> {
|
||||
if self.manifests.contains_key(name) {
|
||||
return Some(name.to_string());
|
||||
}
|
||||
|
||||
let has_tool = self.manifests.contains_key(&format!("tools/{}", name));
|
||||
let has_channel = self.manifests.contains_key(&format!("channels/{}", name));
|
||||
|
||||
match (has_tool, has_channel) {
|
||||
(true, true) => None, // ambiguous
|
||||
(true, false) => Some(format!("tools/{}", name)),
|
||||
(false, true) => Some(format!("channels/{}", name)),
|
||||
(false, false) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Search manifests by query string (matches name, display_name, description, keywords).
|
||||
pub fn search(&self, query: &str) -> Vec<&ExtensionManifest> {
|
||||
let query_lower = query.to_lowercase();
|
||||
let tokens: Vec<&str> = query_lower.split_whitespace().collect();
|
||||
|
||||
let mut scored: Vec<(&ExtensionManifest, usize)> = self
|
||||
.manifests
|
||||
.values()
|
||||
.filter_map(|m| {
|
||||
let score = Self::score_manifest(m, &tokens);
|
||||
if score > 0 { Some((m, score)) } else { None }
|
||||
})
|
||||
.collect();
|
||||
|
||||
scored.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.name.cmp(&b.0.name)));
|
||||
scored.into_iter().map(|(m, _)| m).collect()
|
||||
}
|
||||
|
||||
fn score_manifest(manifest: &ExtensionManifest, tokens: &[&str]) -> usize {
|
||||
let mut score = 0;
|
||||
let name_lower = manifest.name.to_lowercase();
|
||||
let display_lower = manifest.display_name.to_lowercase();
|
||||
let desc_lower = manifest.description.to_lowercase();
|
||||
|
||||
for token in tokens {
|
||||
if name_lower == *token {
|
||||
score += 10;
|
||||
} else if name_lower.contains(token) {
|
||||
score += 5;
|
||||
}
|
||||
|
||||
if display_lower == *token {
|
||||
score += 8;
|
||||
} else if display_lower.contains(token) {
|
||||
score += 4;
|
||||
}
|
||||
|
||||
if desc_lower.contains(token) {
|
||||
score += 2;
|
||||
}
|
||||
|
||||
for kw in &manifest.keywords {
|
||||
if kw.to_lowercase() == *token {
|
||||
score += 6;
|
||||
} else if kw.to_lowercase().contains(token) {
|
||||
score += 3;
|
||||
}
|
||||
}
|
||||
|
||||
for tag in &manifest.tags {
|
||||
if tag.to_lowercase() == *token {
|
||||
score += 4;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
score
|
||||
}
|
||||
|
||||
/// Get a bundle definition by name.
|
||||
pub fn get_bundle(&self, name: &str) -> Option<&BundleDefinition> {
|
||||
self.bundles.get(name)
|
||||
}
|
||||
|
||||
/// List all bundle names.
|
||||
pub fn bundle_names(&self) -> Vec<&str> {
|
||||
let mut names: Vec<_> = self.bundles.keys().map(|s| s.as_str()).collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Resolve a bundle into its constituent manifests.
|
||||
/// Returns the manifests and any extension keys that couldn't be found.
|
||||
pub fn resolve_bundle(
|
||||
&self,
|
||||
bundle_name: &str,
|
||||
) -> Result<(Vec<&ExtensionManifest>, Vec<String>), RegistryError> {
|
||||
let bundle = self
|
||||
.bundles
|
||||
.get(bundle_name)
|
||||
.ok_or_else(|| RegistryError::BundleNotFound(bundle_name.to_string()))?;
|
||||
|
||||
let mut found = Vec::new();
|
||||
let mut missing = Vec::new();
|
||||
|
||||
for ext_key in &bundle.extensions {
|
||||
if let Some(manifest) = self.manifests.get(ext_key) {
|
||||
found.push(manifest);
|
||||
} else {
|
||||
missing.push(ext_key.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((found, missing))
|
||||
}
|
||||
|
||||
/// Check if a name refers to a bundle rather than an individual extension.
|
||||
pub fn is_bundle(&self, name: &str) -> bool {
|
||||
self.bundles.contains_key(name)
|
||||
}
|
||||
|
||||
/// Resolve a name to either a single manifest or the manifests in a bundle.
|
||||
/// Returns (manifests, bundle_definition_if_bundle).
|
||||
pub fn resolve(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<(Vec<&ExtensionManifest>, Option<&BundleDefinition>), RegistryError> {
|
||||
// Check bundle first
|
||||
if let Some(bundle) = self.bundles.get(name) {
|
||||
let (manifests, missing) = self.resolve_bundle(name)?;
|
||||
if !missing.is_empty() {
|
||||
tracing::warn!(
|
||||
"Bundle '{}' references missing extensions: {:?}",
|
||||
name,
|
||||
missing
|
||||
);
|
||||
}
|
||||
return Ok((manifests, Some(bundle)));
|
||||
}
|
||||
|
||||
// Single extension (use get_strict to catch ambiguous bare names)
|
||||
let manifest = self.get_strict(name)?;
|
||||
Ok((vec![manifest], None))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
|
||||
fn create_test_registry(dir: &Path) {
|
||||
let tools_dir = dir.join("tools");
|
||||
let channels_dir = dir.join("channels");
|
||||
fs::create_dir_all(&tools_dir).unwrap();
|
||||
fs::create_dir_all(&channels_dir).unwrap();
|
||||
|
||||
fs::write(
|
||||
tools_dir.join("slack.json"),
|
||||
r#"{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Post messages via Slack API",
|
||||
"keywords": ["messaging", "chat"],
|
||||
"source": {
|
||||
"dir": "tools-src/slack",
|
||||
"capabilities": "slack-tool.capabilities.json",
|
||||
"crate_name": "slack-tool"
|
||||
},
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token"]
|
||||
},
|
||||
"tags": ["default", "messaging"]
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
fs::write(
|
||||
tools_dir.join("github.json"),
|
||||
r#"{
|
||||
"name": "github",
|
||||
"display_name": "GitHub",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "GitHub integration for issues and PRs",
|
||||
"keywords": ["code", "git"],
|
||||
"source": {
|
||||
"dir": "tools-src/github",
|
||||
"capabilities": "github-tool.capabilities.json",
|
||||
"crate_name": "github-tool"
|
||||
},
|
||||
"tags": ["default", "development"]
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
fs::write(
|
||||
channels_dir.join("telegram.json"),
|
||||
r#"{
|
||||
"name": "telegram",
|
||||
"display_name": "Telegram",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Telegram Bot API channel",
|
||||
"source": {
|
||||
"dir": "channels-src/telegram",
|
||||
"capabilities": "telegram.capabilities.json",
|
||||
"crate_name": "telegram-channel"
|
||||
},
|
||||
"tags": ["messaging"]
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
fs::write(
|
||||
dir.join("_bundles.json"),
|
||||
r#"{
|
||||
"bundles": {
|
||||
"default": {
|
||||
"display_name": "Recommended",
|
||||
"extensions": ["tools/slack", "tools/github", "channels/telegram"]
|
||||
},
|
||||
"messaging": {
|
||||
"display_name": "Messaging",
|
||||
"extensions": ["tools/slack", "channels/telegram"],
|
||||
"shared_auth": null
|
||||
}
|
||||
}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_catalog() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
assert_eq!(catalog.all().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_by_kind() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
let tools = catalog.list(Some(ManifestKind::Tool), None);
|
||||
assert_eq!(tools.len(), 2);
|
||||
|
||||
let channels = catalog.list(Some(ManifestKind::Channel), None);
|
||||
assert_eq!(channels.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_list_by_tag() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
let defaults = catalog.list(None, Some("default"));
|
||||
assert_eq!(defaults.len(), 2);
|
||||
|
||||
let messaging = catalog.list(None, Some("messaging"));
|
||||
assert_eq!(messaging.len(), 2); // slack (tool) and telegram (channel) both have "messaging" tag
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_by_name() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
|
||||
// Full key
|
||||
assert!(catalog.get("tools/slack").is_some());
|
||||
|
||||
// Bare name
|
||||
assert!(catalog.get("slack").is_some());
|
||||
assert!(catalog.get("telegram").is_some());
|
||||
|
||||
// Missing
|
||||
assert!(catalog.get("nonexistent").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_search() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
|
||||
let results = catalog.search("slack");
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].name, "slack");
|
||||
|
||||
let results = catalog.search("messaging");
|
||||
assert!(!results.is_empty());
|
||||
|
||||
let results = catalog.search("nonexistent query");
|
||||
assert!(results.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_bundle() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
|
||||
let (manifests, missing) = catalog.resolve_bundle("default").unwrap();
|
||||
assert_eq!(manifests.len(), 3);
|
||||
assert!(missing.is_empty());
|
||||
|
||||
assert!(catalog.resolve_bundle("nonexistent").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resolve_single_or_bundle() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
|
||||
// Single extension
|
||||
let (manifests, bundle) = catalog.resolve("slack").unwrap();
|
||||
assert_eq!(manifests.len(), 1);
|
||||
assert!(bundle.is_none());
|
||||
|
||||
// Bundle
|
||||
let (manifests, bundle) = catalog.resolve("default").unwrap();
|
||||
assert_eq!(manifests.len(), 3);
|
||||
assert!(bundle.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bundle_names() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
create_test_registry(tmp.path());
|
||||
|
||||
let catalog = RegistryCatalog::load(tmp.path()).unwrap();
|
||||
let names = catalog.bundle_names();
|
||||
assert_eq!(names, vec!["default", "messaging"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_directory_not_found() {
|
||||
let result = RegistryCatalog::load(Path::new("/nonexistent/path"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
//! Install extensions from the registry: build-from-source or download pre-built artifacts.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use tokio::fs;
|
||||
|
||||
use crate::registry::catalog::RegistryError;
|
||||
use crate::registry::manifest::{BundleDefinition, ExtensionManifest, ManifestKind};
|
||||
|
||||
/// Result of installing a single extension from the registry.
|
||||
#[derive(Debug)]
|
||||
pub struct InstallOutcome {
|
||||
/// Extension name.
|
||||
pub name: String,
|
||||
/// Whether this is a tool or channel.
|
||||
pub kind: ManifestKind,
|
||||
/// Destination path of the installed WASM binary.
|
||||
pub wasm_path: PathBuf,
|
||||
/// Whether a capabilities file was also installed.
|
||||
pub has_capabilities: bool,
|
||||
/// Any warning messages.
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
/// Handles installing extensions from registry manifests.
|
||||
pub struct RegistryInstaller {
|
||||
/// Root of the repo (parent of `registry/`), used to resolve `source.dir`.
|
||||
repo_root: PathBuf,
|
||||
/// Directory for installed tools (`~/.ironclaw/tools/`).
|
||||
tools_dir: PathBuf,
|
||||
/// Directory for installed channels (`~/.ironclaw/channels/`).
|
||||
channels_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl RegistryInstaller {
|
||||
pub fn new(repo_root: PathBuf, tools_dir: PathBuf, channels_dir: PathBuf) -> Self {
|
||||
Self {
|
||||
repo_root,
|
||||
tools_dir,
|
||||
channels_dir,
|
||||
}
|
||||
}
|
||||
|
||||
/// Default installer using standard paths.
|
||||
pub fn with_defaults(repo_root: PathBuf) -> Self {
|
||||
let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
|
||||
Self {
|
||||
repo_root,
|
||||
tools_dir: home.join(".ironclaw").join("tools"),
|
||||
channels_dir: home.join(".ironclaw").join("channels"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a single extension by building from source.
|
||||
pub async fn install_from_source(
|
||||
&self,
|
||||
manifest: &ExtensionManifest,
|
||||
force: bool,
|
||||
) -> Result<InstallOutcome, RegistryError> {
|
||||
let source_dir = self.repo_root.join(&manifest.source.dir);
|
||||
if !source_dir.exists() {
|
||||
return Err(RegistryError::ManifestRead {
|
||||
path: source_dir.clone(),
|
||||
reason: "source directory does not exist".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let target_dir = match manifest.kind {
|
||||
ManifestKind::Tool => &self.tools_dir,
|
||||
ManifestKind::Channel => &self.channels_dir,
|
||||
};
|
||||
|
||||
fs::create_dir_all(target_dir)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
|
||||
// Use manifest.name for installed filenames so discovery, auth, and
|
||||
// CLI commands (`ironclaw tool auth <name>`) all agree on the stem.
|
||||
let target_wasm = target_dir.join(format!("{}.wasm", manifest.name));
|
||||
|
||||
// Check if already exists
|
||||
if target_wasm.exists() && !force {
|
||||
return Err(RegistryError::AlreadyInstalled {
|
||||
name: manifest.name.clone(),
|
||||
path: target_wasm,
|
||||
});
|
||||
}
|
||||
|
||||
// Build the WASM component
|
||||
println!(
|
||||
"Building {} '{}' from {}...",
|
||||
manifest.kind,
|
||||
manifest.display_name,
|
||||
source_dir.display()
|
||||
);
|
||||
let crate_name = &manifest.source.crate_name;
|
||||
let wasm_path = build_wasm_component(&source_dir, crate_name)
|
||||
.await
|
||||
.map_err(|e| RegistryError::ManifestRead {
|
||||
path: source_dir.clone(),
|
||||
reason: format!("build failed: {}", e),
|
||||
})?;
|
||||
|
||||
// Copy WASM binary
|
||||
println!(" Installing to {}", target_wasm.display());
|
||||
fs::copy(&wasm_path, &target_wasm)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
|
||||
// Copy capabilities file
|
||||
let caps_source = source_dir.join(&manifest.source.capabilities);
|
||||
let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name));
|
||||
let has_capabilities = if caps_source.exists() {
|
||||
fs::copy(&caps_source, &target_caps)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
let mut warnings = Vec::new();
|
||||
if !has_capabilities {
|
||||
warnings.push(format!(
|
||||
"No capabilities file found at {}",
|
||||
caps_source.display()
|
||||
));
|
||||
}
|
||||
|
||||
Ok(InstallOutcome {
|
||||
name: manifest.name.clone(),
|
||||
kind: manifest.kind,
|
||||
wasm_path: target_wasm,
|
||||
has_capabilities,
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
|
||||
/// Download and install a pre-built artifact.
|
||||
pub async fn install_from_artifact(
|
||||
&self,
|
||||
manifest: &ExtensionManifest,
|
||||
force: bool,
|
||||
) -> Result<InstallOutcome, RegistryError> {
|
||||
let artifact = manifest.artifacts.get("wasm32-wasip2").ok_or_else(|| {
|
||||
RegistryError::ExtensionNotFound(format!(
|
||||
"No wasm32-wasip2 artifact for '{}'",
|
||||
manifest.name
|
||||
))
|
||||
})?;
|
||||
|
||||
let url = artifact.url.as_ref().ok_or_else(|| {
|
||||
RegistryError::ExtensionNotFound(format!(
|
||||
"No artifact URL for '{}'. Use --build to build from source.",
|
||||
manifest.name
|
||||
))
|
||||
})?;
|
||||
|
||||
let expected_sha = artifact.sha256.as_ref().ok_or_else(|| {
|
||||
RegistryError::ExtensionNotFound(format!(
|
||||
"No SHA256 hash for '{}'. Cannot verify download.",
|
||||
manifest.name
|
||||
))
|
||||
})?;
|
||||
|
||||
let target_dir = match manifest.kind {
|
||||
ManifestKind::Tool => &self.tools_dir,
|
||||
ManifestKind::Channel => &self.channels_dir,
|
||||
};
|
||||
|
||||
fs::create_dir_all(target_dir)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
|
||||
let target_wasm = target_dir.join(format!("{}.wasm", manifest.name));
|
||||
|
||||
if target_wasm.exists() && !force {
|
||||
return Err(RegistryError::AlreadyInstalled {
|
||||
name: manifest.name.clone(),
|
||||
path: target_wasm,
|
||||
});
|
||||
}
|
||||
|
||||
// Download
|
||||
println!(
|
||||
"Downloading {} '{}'...",
|
||||
manifest.kind, manifest.display_name
|
||||
);
|
||||
let response = reqwest::get(url)
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!("request failed: {}", e),
|
||||
})?;
|
||||
|
||||
let response = response
|
||||
.error_for_status()
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let bytes = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!("failed to read body: {}", e),
|
||||
})?;
|
||||
|
||||
// Verify SHA256
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
let actual_sha = format!("{:x}", hasher.finalize());
|
||||
|
||||
if actual_sha != *expected_sha {
|
||||
return Err(RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!(
|
||||
"SHA256 mismatch: expected {}, got {}",
|
||||
expected_sha, actual_sha
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
// Write file
|
||||
fs::write(&target_wasm, &bytes)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
|
||||
// Copy capabilities from source dir (still needed even for pre-built artifacts).
|
||||
// NOTE: This requires the source tree to be present. When pre-built artifact
|
||||
// distribution is implemented, capabilities should be bundled with the artifact
|
||||
// or fetched from a separate URL.
|
||||
let caps_source = self
|
||||
.repo_root
|
||||
.join(&manifest.source.dir)
|
||||
.join(&manifest.source.capabilities);
|
||||
let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name));
|
||||
let has_capabilities = if caps_source.exists() {
|
||||
fs::copy(&caps_source, &target_caps)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
println!(" Installed to {}", target_wasm.display());
|
||||
|
||||
Ok(InstallOutcome {
|
||||
name: manifest.name.clone(),
|
||||
kind: manifest.kind,
|
||||
wasm_path: target_wasm,
|
||||
has_capabilities,
|
||||
warnings: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Install a single manifest, choosing build vs download based on artifact availability and flags.
|
||||
pub async fn install(
|
||||
&self,
|
||||
manifest: &ExtensionManifest,
|
||||
force: bool,
|
||||
prefer_build: bool,
|
||||
) -> Result<InstallOutcome, RegistryError> {
|
||||
let has_artifact = manifest
|
||||
.artifacts
|
||||
.get("wasm32-wasip2")
|
||||
.and_then(|a| a.url.as_ref())
|
||||
.is_some();
|
||||
|
||||
if prefer_build || !has_artifact {
|
||||
self.install_from_source(manifest, force).await
|
||||
} else {
|
||||
self.install_from_artifact(manifest, force).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Install all extensions in a bundle.
|
||||
/// Returns the outcomes and any shared auth hints.
|
||||
pub async fn install_bundle(
|
||||
&self,
|
||||
manifests: &[&ExtensionManifest],
|
||||
bundle: &BundleDefinition,
|
||||
force: bool,
|
||||
prefer_build: bool,
|
||||
) -> (Vec<InstallOutcome>, Vec<String>) {
|
||||
let mut outcomes = Vec::new();
|
||||
let mut errors = Vec::new();
|
||||
|
||||
for manifest in manifests {
|
||||
match self.install(manifest, force, prefer_build).await {
|
||||
Ok(outcome) => outcomes.push(outcome),
|
||||
Err(e) => errors.push(format!("{}: {}", manifest.name, e)),
|
||||
}
|
||||
}
|
||||
|
||||
// Collect auth hints
|
||||
let mut auth_hints = Vec::new();
|
||||
if let Some(shared) = &bundle.shared_auth {
|
||||
auth_hints.push(format!(
|
||||
"Bundle uses shared auth '{}'. Run `ironclaw tool auth <any-member>` to authenticate all members.",
|
||||
shared
|
||||
));
|
||||
}
|
||||
|
||||
// Collect unique auth providers that need setup
|
||||
let mut seen_providers = std::collections::HashSet::new();
|
||||
for manifest in manifests {
|
||||
if let Some(auth) = &manifest.auth_summary {
|
||||
let key = auth
|
||||
.shared_auth
|
||||
.as_deref()
|
||||
.unwrap_or(manifest.name.as_str());
|
||||
if seen_providers.insert(key.to_string())
|
||||
&& let Some(url) = &auth.setup_url
|
||||
{
|
||||
auth_hints.push(format!(
|
||||
" {} ({}): {}",
|
||||
auth.provider.as_deref().unwrap_or(&manifest.name),
|
||||
auth.method.as_deref().unwrap_or("manual"),
|
||||
url
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !errors.is_empty() {
|
||||
auth_hints.push(format!(
|
||||
"\nFailed to install {} extension(s):",
|
||||
errors.len()
|
||||
));
|
||||
for err in errors {
|
||||
auth_hints.push(format!(" - {}", err));
|
||||
}
|
||||
}
|
||||
|
||||
(outcomes, auth_hints)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a WASM component from a source directory using `cargo component build --release`.
|
||||
///
|
||||
/// Uses `tokio::process::Command` with inherited stdio so build progress is visible.
|
||||
/// Looks for the specific `{crate_name}.wasm` in the release directory rather than
|
||||
/// picking the first `.wasm` file found.
|
||||
async fn build_wasm_component(source_dir: &Path, crate_name: &str) -> anyhow::Result<PathBuf> {
|
||||
use tokio::process::Command;
|
||||
|
||||
// Check cargo-component availability
|
||||
let check = Command::new("cargo")
|
||||
.args(["component", "--version"])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.await;
|
||||
|
||||
if check.is_err() || !check.as_ref().map(|s| s.success()).unwrap_or(false) {
|
||||
anyhow::bail!("cargo-component not found. Install with: cargo install cargo-component");
|
||||
}
|
||||
|
||||
// Use status() with inherited stdio so build output streams to the terminal.
|
||||
let status = Command::new("cargo")
|
||||
.current_dir(source_dir)
|
||||
.args(["component", "build", "--release"])
|
||||
.status()
|
||||
.await?;
|
||||
|
||||
if !status.success() {
|
||||
anyhow::bail!("Build failed (exit code: {})", status);
|
||||
}
|
||||
|
||||
// Look for the specific crate's WASM file (Cargo uses underscores in artifact names).
|
||||
let wasm_filename = format!("{}.wasm", crate_name.replace('-', "_"));
|
||||
let target_base = source_dir.join("target");
|
||||
let candidates = [
|
||||
"wasm32-wasip1",
|
||||
"wasm32-wasip2",
|
||||
"wasm32-wasi",
|
||||
"wasm32-unknown-unknown",
|
||||
];
|
||||
|
||||
for target in &candidates {
|
||||
let wasm_path = target_base
|
||||
.join(target)
|
||||
.join("release")
|
||||
.join(&wasm_filename);
|
||||
if wasm_path.exists() {
|
||||
return Ok(wasm_path);
|
||||
}
|
||||
}
|
||||
|
||||
anyhow::bail!(
|
||||
"Could not find {} in {}/target/*/release/",
|
||||
wasm_filename,
|
||||
source_dir.display()
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_installer_creation() {
|
||||
let installer = RegistryInstaller::new(
|
||||
PathBuf::from("/repo"),
|
||||
PathBuf::from("/home/.ironclaw/tools"),
|
||||
PathBuf::from("/home/.ironclaw/channels"),
|
||||
);
|
||||
assert_eq!(installer.repo_root, PathBuf::from("/repo"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
//! Serde structs for extension registry manifests.
|
||||
//!
|
||||
//! Each manifest describes a single extension (tool or channel) with its source
|
||||
//! location, build artifacts, authentication requirements, and tags.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::extensions::{AuthHint, ExtensionKind, ExtensionSource, RegistryEntry};
|
||||
|
||||
/// A single extension manifest loaded from `registry/{tools,channels}/<name>.json`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ExtensionManifest {
|
||||
/// Unique identifier (matches crate name stem, e.g. "slack").
|
||||
pub name: String,
|
||||
|
||||
/// Human-readable name (e.g. "Slack").
|
||||
pub display_name: String,
|
||||
|
||||
/// Whether this is a tool or channel.
|
||||
pub kind: ManifestKind,
|
||||
|
||||
/// Semver version from Cargo.toml.
|
||||
pub version: String,
|
||||
|
||||
/// One-line description.
|
||||
pub description: String,
|
||||
|
||||
/// Search keywords beyond the name.
|
||||
#[serde(default)]
|
||||
pub keywords: Vec<String>,
|
||||
|
||||
/// Source code location and build info.
|
||||
pub source: SourceSpec,
|
||||
|
||||
/// Pre-built binary artifacts keyed by target triple.
|
||||
#[serde(default)]
|
||||
pub artifacts: std::collections::HashMap<String, ArtifactSpec>,
|
||||
|
||||
/// Summary of authentication requirements.
|
||||
#[serde(default)]
|
||||
pub auth_summary: Option<AuthSummary>,
|
||||
|
||||
/// Tags for filtering (e.g. "default", "messaging", "google").
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
}
|
||||
|
||||
/// Extension kind as declared in manifests.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ManifestKind {
|
||||
Tool,
|
||||
Channel,
|
||||
}
|
||||
|
||||
impl From<ManifestKind> for ExtensionKind {
|
||||
fn from(kind: ManifestKind) -> Self {
|
||||
match kind {
|
||||
ManifestKind::Tool => ExtensionKind::WasmTool,
|
||||
ManifestKind::Channel => ExtensionKind::WasmChannel,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ManifestKind {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ManifestKind::Tool => write!(f, "tool"),
|
||||
ManifestKind::Channel => write!(f, "channel"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Source code location for building from source.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SourceSpec {
|
||||
/// Path relative to repo root (e.g. "tools-src/slack").
|
||||
pub dir: String,
|
||||
|
||||
/// Capabilities filename relative to source dir.
|
||||
pub capabilities: String,
|
||||
|
||||
/// Rust crate name for `cargo component build`.
|
||||
pub crate_name: String,
|
||||
}
|
||||
|
||||
/// A pre-built binary artifact.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ArtifactSpec {
|
||||
/// Download URL (null until release).
|
||||
pub url: Option<String>,
|
||||
|
||||
/// Hex SHA256 of the WASM binary (null until release).
|
||||
pub sha256: Option<String>,
|
||||
}
|
||||
|
||||
/// Summary of authentication requirements extracted from capabilities.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AuthSummary {
|
||||
/// Auth method: "oauth", "manual", or "none".
|
||||
#[serde(default)]
|
||||
pub method: Option<String>,
|
||||
|
||||
/// Display name for the auth provider (e.g. "Google", "Slack").
|
||||
#[serde(default)]
|
||||
pub provider: Option<String>,
|
||||
|
||||
/// Secret names required by this extension.
|
||||
#[serde(default)]
|
||||
pub secrets: Vec<String>,
|
||||
|
||||
/// If this extension shares auth with others (e.g. all Google tools share
|
||||
/// `google_oauth_token`), this is the shared secret name.
|
||||
#[serde(default)]
|
||||
pub shared_auth: Option<String>,
|
||||
|
||||
/// URL where users can set up credentials.
|
||||
#[serde(default)]
|
||||
pub setup_url: Option<String>,
|
||||
}
|
||||
|
||||
/// Bundle definition grouping related extensions.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BundleDefinition {
|
||||
/// Human-readable name.
|
||||
pub display_name: String,
|
||||
|
||||
/// Description of what this bundle contains.
|
||||
#[serde(default)]
|
||||
pub description: Option<String>,
|
||||
|
||||
/// Extension references as "tools/<name>" or "channels/<name>".
|
||||
pub extensions: Vec<String>,
|
||||
|
||||
/// Shared auth secret across bundle members (if any).
|
||||
#[serde(default)]
|
||||
pub shared_auth: Option<String>,
|
||||
}
|
||||
|
||||
/// Top-level structure of `_bundles.json`.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BundlesFile {
|
||||
pub bundles: std::collections::HashMap<String, BundleDefinition>,
|
||||
}
|
||||
|
||||
impl ExtensionManifest {
|
||||
/// Convert this manifest into a [`RegistryEntry`] for use with the in-chat
|
||||
/// extension discovery system.
|
||||
pub fn to_registry_entry(&self) -> RegistryEntry {
|
||||
let source = ExtensionSource::WasmBuildable {
|
||||
repo_url: self.source.dir.clone(),
|
||||
build_dir: Some(self.source.dir.clone()),
|
||||
};
|
||||
|
||||
let auth_hint = match self.auth_summary.as_ref().and_then(|a| a.method.as_deref()) {
|
||||
Some("oauth") => AuthHint::CapabilitiesAuth,
|
||||
Some("manual") => AuthHint::CapabilitiesAuth,
|
||||
Some("none") | None => AuthHint::None,
|
||||
Some(_) => AuthHint::CapabilitiesAuth,
|
||||
};
|
||||
|
||||
RegistryEntry {
|
||||
name: self.name.clone(),
|
||||
display_name: self.display_name.clone(),
|
||||
kind: self.kind.into(),
|
||||
description: self.description.clone(),
|
||||
keywords: self.keywords.clone(),
|
||||
source,
|
||||
auth_hint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_tool_manifest() {
|
||||
let json = r#"{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Post messages via Slack API",
|
||||
"keywords": ["messaging"],
|
||||
"source": {
|
||||
"dir": "tools-src/slack",
|
||||
"capabilities": "slack-tool.capabilities.json",
|
||||
"crate_name": "slack-tool"
|
||||
},
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": { "url": null, "sha256": null }
|
||||
},
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://api.slack.com/apps"
|
||||
},
|
||||
"tags": ["default", "messaging"]
|
||||
}"#;
|
||||
|
||||
let manifest: ExtensionManifest = serde_json::from_str(json).expect("parse manifest");
|
||||
assert_eq!(manifest.name, "slack");
|
||||
assert_eq!(manifest.kind, ManifestKind::Tool);
|
||||
assert_eq!(manifest.version, "0.1.0");
|
||||
assert!(manifest.tags.contains(&"default".to_string()));
|
||||
|
||||
let entry = manifest.to_registry_entry();
|
||||
assert_eq!(entry.kind, ExtensionKind::WasmTool);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_channel_manifest() {
|
||||
let json = r#"{
|
||||
"name": "telegram",
|
||||
"display_name": "Telegram",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Telegram Bot API channel",
|
||||
"source": {
|
||||
"dir": "channels-src/telegram",
|
||||
"capabilities": "telegram.capabilities.json",
|
||||
"crate_name": "telegram-channel"
|
||||
},
|
||||
"tags": ["messaging"]
|
||||
}"#;
|
||||
|
||||
let manifest: ExtensionManifest = serde_json::from_str(json).expect("parse manifest");
|
||||
assert_eq!(manifest.kind, ManifestKind::Channel);
|
||||
assert!(manifest.auth_summary.is_none());
|
||||
assert!(manifest.artifacts.is_empty());
|
||||
|
||||
let entry = manifest.to_registry_entry();
|
||||
assert_eq!(entry.kind, ExtensionKind::WasmChannel);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_bundles() {
|
||||
let json = r#"{
|
||||
"bundles": {
|
||||
"google": {
|
||||
"display_name": "Google Suite",
|
||||
"description": "All Google tools",
|
||||
"extensions": ["tools/gmail", "tools/google-calendar"],
|
||||
"shared_auth": "google_oauth_token"
|
||||
},
|
||||
"default": {
|
||||
"display_name": "Recommended Set",
|
||||
"extensions": ["tools/github", "tools/slack"]
|
||||
}
|
||||
}
|
||||
}"#;
|
||||
|
||||
let bundles: BundlesFile = serde_json::from_str(json).expect("parse bundles");
|
||||
assert_eq!(bundles.bundles.len(), 2);
|
||||
assert_eq!(
|
||||
bundles.bundles["google"].shared_auth.as_deref(),
|
||||
Some("google_oauth_token")
|
||||
);
|
||||
assert!(bundles.bundles["default"].shared_auth.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_manifest_kind_display() {
|
||||
assert_eq!(ManifestKind::Tool.to_string(), "tool");
|
||||
assert_eq!(ManifestKind::Channel.to_string(), "channel");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Extension registry: metadata catalog for tools and channels.
|
||||
//!
|
||||
//! The registry provides a central index of all available extensions (WASM tools
|
||||
//! and channels) with their source locations, build artifacts, authentication
|
||||
//! requirements, and grouping via bundles.
|
||||
//!
|
||||
//! ```text
|
||||
//! registry/
|
||||
//! ├── tools/ <- One JSON manifest per tool
|
||||
//! ├── channels/ <- One JSON manifest per channel
|
||||
//! └── _bundles.json <- Bundle definitions (google, messaging, default)
|
||||
//! ```
|
||||
|
||||
pub mod catalog;
|
||||
pub mod installer;
|
||||
pub mod manifest;
|
||||
|
||||
pub use catalog::{RegistryCatalog, RegistryError};
|
||||
pub use installer::RegistryInstaller;
|
||||
pub use manifest::{
|
||||
ArtifactSpec, AuthSummary, BundleDefinition, BundlesFile, ExtensionManifest, ManifestKind,
|
||||
SourceSpec,
|
||||
};
|
||||
+103
@@ -1229,4 +1229,107 @@ mod tests {
|
||||
assert_eq!(s.tunnel.cf_token, Some("cf_tok_xyz".to_string()));
|
||||
assert!(s.tunnel.ts_funnel);
|
||||
}
|
||||
|
||||
/// Simulates the wizard recovery scenario:
|
||||
///
|
||||
/// 1. A prior partial run saved steps 1-4 to the DB
|
||||
/// 2. User re-runs the wizard, Step 1 sets a new database_url
|
||||
/// 3. Prior settings are loaded from the DB
|
||||
/// 4. Step 1's fresh choices must win over stale DB values
|
||||
///
|
||||
/// This tests the ordering: load DB → merge_from(step1_overrides).
|
||||
#[test]
|
||||
fn wizard_recovery_step1_overrides_stale_db() {
|
||||
// Simulate prior partial run (steps 1-4 completed):
|
||||
let prior_run = Settings {
|
||||
database_backend: Some("postgres".to_string()),
|
||||
database_url: Some("postgres://old-host/ironclaw".to_string()),
|
||||
llm_backend: Some("anthropic".to_string()),
|
||||
selected_model: Some("claude-sonnet-4-5".to_string()),
|
||||
embeddings: EmbeddingsSettings {
|
||||
enabled: true,
|
||||
provider: "openai".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Save to DB and reload (simulates persistence round-trip)
|
||||
let db_map = prior_run.to_db_map();
|
||||
let from_db = Settings::from_db_map(&db_map);
|
||||
|
||||
// Step 1 of the new wizard run: user enters a NEW database_url
|
||||
let mut step1_settings = Settings::default();
|
||||
step1_settings.database_backend = Some("postgres".to_string());
|
||||
step1_settings.database_url = Some("postgres://new-host/ironclaw".to_string());
|
||||
|
||||
// Wizard flow: load DB → merge_from(step1_overrides)
|
||||
let mut current = step1_settings.clone();
|
||||
// try_load_existing_settings: merge DB into current
|
||||
current.merge_from(&from_db);
|
||||
// Re-apply Step 1 choices on top
|
||||
current.merge_from(&step1_settings);
|
||||
|
||||
// Step 1's fresh database_url wins over stale DB value
|
||||
assert_eq!(
|
||||
current.database_url,
|
||||
Some("postgres://new-host/ironclaw".to_string()),
|
||||
"Step 1 fresh choice must override stale DB value"
|
||||
);
|
||||
|
||||
// Prior run's steps 2-4 settings are preserved
|
||||
assert_eq!(
|
||||
current.llm_backend,
|
||||
Some("anthropic".to_string()),
|
||||
"Prior run's LLM backend must be recovered"
|
||||
);
|
||||
assert_eq!(
|
||||
current.selected_model,
|
||||
Some("claude-sonnet-4-5".to_string()),
|
||||
"Prior run's model must be recovered"
|
||||
);
|
||||
assert!(
|
||||
current.embeddings.enabled,
|
||||
"Prior run's embeddings setting must be recovered"
|
||||
);
|
||||
}
|
||||
|
||||
/// Verifies that persisting defaults doesn't clobber prior settings
|
||||
/// when the merge ordering is correct.
|
||||
#[test]
|
||||
fn wizard_recovery_defaults_dont_clobber_prior() {
|
||||
// Prior run saved non-default settings
|
||||
let prior_run = Settings {
|
||||
llm_backend: Some("openai".to_string()),
|
||||
selected_model: Some("gpt-4o".to_string()),
|
||||
heartbeat: HeartbeatSettings {
|
||||
enabled: true,
|
||||
interval_secs: 900,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let db_map = prior_run.to_db_map();
|
||||
let from_db = Settings::from_db_map(&db_map);
|
||||
|
||||
// New wizard run: Step 1 only sets DB fields (rest is default)
|
||||
let step1 = Settings {
|
||||
database_backend: Some("libsql".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Correct merge ordering
|
||||
let mut current = step1.clone();
|
||||
current.merge_from(&from_db);
|
||||
current.merge_from(&step1);
|
||||
|
||||
// Prior settings preserved (Step 1 doesn't touch these)
|
||||
assert_eq!(current.llm_backend, Some("openai".to_string()));
|
||||
assert_eq!(current.selected_model, Some("gpt-4o".to_string()));
|
||||
assert!(current.heartbeat.enabled);
|
||||
assert_eq!(current.heartbeat.interval_secs, 900);
|
||||
|
||||
// Step 1's choice applied
|
||||
assert_eq!(current.database_backend, Some("libsql".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
+128
-24
@@ -50,7 +50,7 @@ The `--no-onboard` CLI flag suppresses auto-detection.
|
||||
|
||||
---
|
||||
|
||||
## The 7-Step Wizard
|
||||
## The 8-Step Wizard
|
||||
|
||||
### Overview
|
||||
|
||||
@@ -61,7 +61,8 @@ Step 3: Inference Provider ← skipped if --skip-auth
|
||||
Step 4: Model Selection
|
||||
Step 5: Embeddings
|
||||
Step 6: Channel Configuration
|
||||
Step 7: Background Tasks (heartbeat)
|
||||
Step 7: Extensions (tools)
|
||||
Step 8: Background Tasks (heartbeat)
|
||||
↓
|
||||
save_and_summarize()
|
||||
```
|
||||
@@ -166,7 +167,8 @@ env-var mode or skipped secrets.
|
||||
|
||||
| Provider | Auth Method | Secret Name | Env Var |
|
||||
|----------|-------------|-------------|---------|
|
||||
| NEAR AI | Browser OAuth | (session token) | `NEARAI_SESSION_TOKEN` |
|
||||
| NEAR AI Chat | Browser OAuth or session token | - | `NEARAI_SESSION_TOKEN` |
|
||||
| NEAR AI Cloud | API key | `llm_nearai_api_key` | `NEARAI_API_KEY` |
|
||||
| Anthropic | API key | `anthropic_api_key` | `ANTHROPIC_API_KEY` |
|
||||
| OpenAI | API key | `openai_api_key` | `OPENAI_API_KEY` |
|
||||
| Ollama | None | - | - |
|
||||
@@ -179,8 +181,18 @@ env-var mode or skipped secrets.
|
||||
4. **Cache key in `self.llm_api_key`** for model fetching in Step 4
|
||||
|
||||
**NEAR AI** (`setup_nearai`):
|
||||
- Calls `session_manager.ensure_authenticated()` which opens browser
|
||||
- Session token saved to `~/.ironclaw/session.json`
|
||||
- Calls `session_manager.ensure_authenticated()` which shows the auth menu:
|
||||
- Options 1-2 (GitHub/Google): browser OAuth → **NEAR AI Chat** mode
|
||||
(Responses API at `private.near.ai`, session token auth)
|
||||
- Option 4: NEAR AI Cloud API key → **NEAR AI Cloud** mode
|
||||
(Chat Completions API at `cloud-api.near.ai`, API key auth)
|
||||
- **NEAR AI Chat** path: session token saved to `~/.ironclaw/session.json`.
|
||||
Hosting providers can set `NEARAI_SESSION_TOKEN` env var directly (takes
|
||||
precedence over file-based tokens).
|
||||
- **NEAR AI Cloud** path: `NEARAI_API_KEY` saved to `~/.ironclaw/.env`
|
||||
(bootstrap) and encrypted secrets store (`llm_nearai_api_key`).
|
||||
`LlmConfig::resolve()` auto-selects `ChatCompletions` mode when the
|
||||
API key is present.
|
||||
|
||||
**`self.llm_api_key` caching:** The wizard caches the API key as
|
||||
`Option<SecretString>` so that Step 4 (model fetching) and Step 5
|
||||
@@ -243,13 +255,20 @@ key first, then falls back to the standard env var.
|
||||
```
|
||||
6a. Tunnel setup (if webhook channels needed)
|
||||
6b. Discover WASM channels from ~/.ironclaw/channels/
|
||||
6c. Multi-select: CLI/TUI, HTTP, discovered channels, bundled channels
|
||||
6d. Install missing bundled channels (copy WASM binaries)
|
||||
6e. Initialize SecretsContext (for token storage)
|
||||
6f. Setup HTTP webhook (if selected)
|
||||
6g. Setup each WASM channel (secrets, owner binding)
|
||||
6c. Build channel options: discovered + bundled + registry catalog
|
||||
6d. Multi-select: CLI/TUI, HTTP, all available channels
|
||||
6e. Install missing bundled channels (copy WASM binaries)
|
||||
6f. Install missing registry channels (build from source)
|
||||
6g. Initialize SecretsContext (for token storage)
|
||||
6h. Setup HTTP webhook (if selected)
|
||||
6i. Setup each WASM channel (secrets, owner binding)
|
||||
```
|
||||
|
||||
**Channel sources** (priority order for installation):
|
||||
1. Already installed in `~/.ironclaw/channels/`
|
||||
2. Bundled channels (pre-compiled in `channels-src/`)
|
||||
3. Registry channels (`registry/channels/*.json`, built from source)
|
||||
|
||||
**Tunnel setup** (`setup_tunnel`):
|
||||
- Options: ngrok, Cloudflare Tunnel, localtunnel, custom URL
|
||||
- Validates HTTPS requirement
|
||||
@@ -273,7 +292,33 @@ key first, then falls back to the standard env var.
|
||||
|
||||
---
|
||||
|
||||
### Step 7: Heartbeat
|
||||
### Step 7: Extensions (Tools)
|
||||
|
||||
**Module:** `wizard.rs` → `step_extensions()`
|
||||
|
||||
**Goal:** Install WASM tools from the extension registry.
|
||||
|
||||
**Flow:**
|
||||
1. Load `RegistryCatalog` from `registry/` directory
|
||||
2. If registry not found, print info and skip
|
||||
3. List all tool manifests from the catalog
|
||||
4. Discover already-installed tools in `~/.ironclaw/tools/`
|
||||
5. Multi-select: show all registry tools with display name, auth method,
|
||||
and description. Pre-check tools tagged `"default"` and already installed.
|
||||
6. For each selected tool not yet installed, build from source via
|
||||
`RegistryInstaller::install_from_source()`
|
||||
7. Print consolidated auth hints (deduplicated by provider, e.g. one hint
|
||||
for all Google tools sharing `google_oauth_token`)
|
||||
|
||||
**Registry lookup** (`load_registry_catalog`):
|
||||
Searches for `registry/` directory in order:
|
||||
1. Current working directory
|
||||
2. Next to the executable
|
||||
3. `CARGO_MANIFEST_DIR` (compile-time, dev builds)
|
||||
|
||||
---
|
||||
|
||||
### Step 8: Heartbeat
|
||||
|
||||
**Module:** `wizard.rs` → `step_heartbeat()`
|
||||
|
||||
@@ -338,25 +383,60 @@ heartbeat.enabled = "true"
|
||||
heartbeat.interval_secs = "300"
|
||||
```
|
||||
|
||||
### Incremental Persistence
|
||||
|
||||
Settings are persisted **after every successful step**, not just at the end.
|
||||
This prevents data loss if a later step fails (e.g., the user enters an
|
||||
API key in step 3 but step 5 crashes — they won't need to re-enter it).
|
||||
|
||||
**`persist_after_step()`** is called after each step in `run()` and:
|
||||
1. Writes bootstrap vars to `~/.ironclaw/.env` via `write_bootstrap_env()`
|
||||
2. Writes all current settings to the database via `persist_settings()`
|
||||
3. Silently ignores errors (e.g., if called before Step 1 establishes a DB)
|
||||
|
||||
**`try_load_existing_settings()`** is called after Step 1 establishes a
|
||||
database connection. It loads any previously saved settings from the
|
||||
database using `get_all_settings("default")` → `Settings::from_db_map()`
|
||||
→ `merge_from()`. This recovers progress from prior partial wizard runs.
|
||||
|
||||
**Ordering after Step 1 is critical:**
|
||||
|
||||
```
|
||||
step_database() → sets DB fields in self.settings
|
||||
let step1 = self.settings.clone() → snapshot Step 1 choices
|
||||
try_load_existing_settings() → merge DB values into self.settings
|
||||
self.settings.merge_from(&step1) → re-apply Step 1 (fresh wins over stale)
|
||||
persist_after_step() → save merged state
|
||||
```
|
||||
|
||||
This ordering ensures:
|
||||
- Prior progress (steps 2-7 from a previous partial run) is recovered
|
||||
- Fresh Step 1 choices override stale DB values (not the reverse)
|
||||
- The first DB persist doesn't clobber prior settings with defaults
|
||||
|
||||
### save_and_summarize()
|
||||
|
||||
Final step of the wizard:
|
||||
|
||||
```
|
||||
1. Mark onboard_completed = true
|
||||
2. Write ALL settings to database (try postgres pool, then libSQL backend)
|
||||
3. Write bootstrap vars to ~/.ironclaw/.env:
|
||||
- DATABASE_BACKEND (always)
|
||||
- DATABASE_URL (if postgres)
|
||||
- LIBSQL_PATH (if libsql)
|
||||
- LIBSQL_URL (if turso sync)
|
||||
- LLM_BACKEND (always, when set)
|
||||
- LLM_BASE_URL (if openai_compatible)
|
||||
- OLLAMA_BASE_URL (if ollama)
|
||||
- ONBOARD_COMPLETED (always, "true")
|
||||
2. Call persist_settings() for final write (idempotent — ensures
|
||||
onboard_completed flag is saved)
|
||||
3. Call write_bootstrap_env() for final .env write (idempotent)
|
||||
4. Print configuration summary
|
||||
```
|
||||
|
||||
Bootstrap vars written to `~/.ironclaw/.env`:
|
||||
- `DATABASE_BACKEND` (always)
|
||||
- `DATABASE_URL` (if postgres)
|
||||
- `LIBSQL_PATH` (if libsql)
|
||||
- `LIBSQL_URL` (if turso sync)
|
||||
- `LLM_BACKEND` (always, when set)
|
||||
- `LLM_BASE_URL` (if openai_compatible)
|
||||
- `OLLAMA_BASE_URL` (if ollama)
|
||||
- `NEARAI_API_KEY` (if API key auth path)
|
||||
- `ONBOARD_COMPLETED` (always, "true")
|
||||
|
||||
**Invariant:** Both Layer 1 and Layer 2 must be written. If the database
|
||||
write fails, the wizard returns an error and the `.env` file is not written.
|
||||
|
||||
@@ -464,9 +544,9 @@ anthropic_api_key → encrypted API key
|
||||
| `confirm(label, default)` | `[Y/n]` or `[y/N]` prompt |
|
||||
| `print_header(text)` | Bold section header with underline |
|
||||
| `print_step(n, total, text)` | `[1/7] Step Name` |
|
||||
| `print_success(text)` | Green checkmark prefix |
|
||||
| `print_error(text)` | Red X prefix |
|
||||
| `print_info(text)` | Blue info prefix |
|
||||
| `print_success(text)` | Green `✓` prefix (ANSI color), message in default color |
|
||||
| `print_error(text)` | Red `✗` prefix (ANSI color), message in default color |
|
||||
| `print_info(text)` | Blue `ℹ` prefix (ANSI color), message in default color |
|
||||
|
||||
`select_many` uses `crossterm` raw mode for arrow key navigation.
|
||||
Must properly restore terminal state on all exit paths.
|
||||
@@ -489,6 +569,30 @@ Must properly restore terminal state on all exit paths.
|
||||
- May need `gnome-keyring` daemon running
|
||||
- Collection unlock may prompt for password
|
||||
|
||||
### Remote Server Authentication
|
||||
|
||||
On remote/VPS servers, the browser-based OAuth flow for NEAR AI may not
|
||||
work because `http://127.0.0.1:9876` is unreachable from the user's
|
||||
local browser.
|
||||
|
||||
**Solutions:**
|
||||
|
||||
1. **NEAR AI Cloud API key (option 4 in auth menu):** Get an API key
|
||||
from `https://cloud.near.ai` and paste it into the terminal. No
|
||||
local listener is needed. The key is saved to `~/.ironclaw/.env`
|
||||
and the encrypted secrets store. Uses the OpenAI-compatible
|
||||
ChatCompletions API mode.
|
||||
|
||||
2. **Custom callback URL:** Set `IRONCLAW_OAUTH_CALLBACK_URL` to a
|
||||
publicly accessible URL (e.g., via SSH tunnel or reverse proxy) that
|
||||
forwards to port 9876 on the server:
|
||||
```bash
|
||||
export IRONCLAW_OAUTH_CALLBACK_URL=https://myserver.example.com:9876
|
||||
```
|
||||
|
||||
The `callback_url()` function in `oauth_defaults.rs` checks this env var
|
||||
and falls back to `http://127.0.0.1:{OAUTH_CALLBACK_PORT}`.
|
||||
|
||||
### URL Passwords
|
||||
|
||||
- `#` is common in URL-encoded passwords (`%23` decoded)
|
||||
|
||||
+46
-4
@@ -363,12 +363,54 @@ pub fn setup_tunnel(settings: &Settings) -> Result<TunnelSettings, ChannelSetupE
|
||||
// Show existing config
|
||||
let has_existing = settings.tunnel.public_url.is_some() || settings.tunnel.provider.is_some();
|
||||
if has_existing {
|
||||
if let Some(ref url) = settings.tunnel.public_url {
|
||||
print_info(&format!("Existing static tunnel URL: {}", url));
|
||||
println!();
|
||||
print_info("Current tunnel configuration:");
|
||||
let t = &settings.tunnel;
|
||||
match t.provider.as_deref() {
|
||||
Some("ngrok") => {
|
||||
print_info(" Provider: ngrok");
|
||||
if let Some(ref domain) = t.ngrok_domain {
|
||||
print_info(&format!(" Domain: {}", domain));
|
||||
}
|
||||
if t.ngrok_token.is_some() {
|
||||
print_info(" Auth: token configured");
|
||||
}
|
||||
}
|
||||
Some("cloudflare") => {
|
||||
print_info(" Provider: Cloudflare Tunnel");
|
||||
if t.cf_token.is_some() {
|
||||
print_info(" Auth: token configured");
|
||||
}
|
||||
}
|
||||
Some("tailscale") => {
|
||||
let mode = if t.ts_funnel {
|
||||
"Funnel (public)"
|
||||
} else {
|
||||
"Serve (tailnet-only)"
|
||||
};
|
||||
print_info(&format!(" Provider: Tailscale {}", mode));
|
||||
if let Some(ref hostname) = t.ts_hostname {
|
||||
print_info(&format!(" Hostname: {}", hostname));
|
||||
}
|
||||
}
|
||||
Some("custom") => {
|
||||
print_info(" Provider: Custom command");
|
||||
if let Some(ref cmd) = t.custom_command {
|
||||
print_info(&format!(" Command: {}", cmd));
|
||||
}
|
||||
if let Some(ref url) = t.custom_health_url {
|
||||
print_info(&format!(" Health: {}", url));
|
||||
}
|
||||
}
|
||||
Some(other) => {
|
||||
print_info(&format!(" Provider: {}", other));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if let Some(ref provider) = settings.tunnel.provider {
|
||||
print_info(&format!("Existing managed provider: {}", provider));
|
||||
if let Some(ref url) = t.public_url {
|
||||
print_info(&format!(" URL: {}", url));
|
||||
}
|
||||
println!();
|
||||
if !confirm("Change tunnel configuration?", false)? {
|
||||
return Ok(settings.tunnel.clone());
|
||||
}
|
||||
|
||||
+2
-1
@@ -7,7 +7,8 @@
|
||||
//! 4. Model selection
|
||||
//! 5. Embeddings
|
||||
//! 6. Channel configuration (HTTP, Telegram, etc.)
|
||||
//! 7. Heartbeat (background tasks)
|
||||
//! 7. Extensions (tool installation from registry)
|
||||
//! 8. Heartbeat (background tasks)
|
||||
//!
|
||||
//! # Example
|
||||
//!
|
||||
|
||||
+29
-6
@@ -293,19 +293,31 @@ pub fn print_step(current: usize, total: usize, name: &str) {
|
||||
println!();
|
||||
}
|
||||
|
||||
/// Print a success message with checkmark.
|
||||
/// Print a success message with green checkmark.
|
||||
pub fn print_success(message: &str) {
|
||||
println!("✓ {}", message);
|
||||
let mut stdout = io::stdout();
|
||||
let _ = execute!(stdout, SetForegroundColor(Color::Green));
|
||||
print!("✓");
|
||||
let _ = execute!(stdout, ResetColor);
|
||||
println!(" {}", message);
|
||||
}
|
||||
|
||||
/// Print an error message.
|
||||
/// Print an error message with red X.
|
||||
pub fn print_error(message: &str) {
|
||||
eprintln!("✗ {}", message);
|
||||
let mut stderr = io::stderr();
|
||||
let _ = execute!(stderr, SetForegroundColor(Color::Red));
|
||||
eprint!("✗");
|
||||
let _ = execute!(stderr, ResetColor);
|
||||
eprintln!(" {}", message);
|
||||
}
|
||||
|
||||
/// Print an info message.
|
||||
/// Print an info message with blue info icon.
|
||||
pub fn print_info(message: &str) {
|
||||
println!(" {}", message);
|
||||
let mut stdout = io::stdout();
|
||||
let _ = execute!(stdout, SetForegroundColor(Color::Blue));
|
||||
print!("ℹ");
|
||||
let _ = execute!(stdout, ResetColor);
|
||||
println!(" {}", message);
|
||||
}
|
||||
|
||||
/// Read a simple line of input with a prompt.
|
||||
@@ -358,4 +370,15 @@ mod tests {
|
||||
super::print_step(1, 3, "Test Step");
|
||||
super::print_step(3, 3, "Final Step");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_print_functions_do_not_panic() {
|
||||
super::print_success("operation completed");
|
||||
super::print_error("something went wrong");
|
||||
super::print_info("here is some information");
|
||||
// Also test with empty strings
|
||||
super::print_success("");
|
||||
super::print_error("");
|
||||
super::print_info("");
|
||||
}
|
||||
}
|
||||
|
||||
+690
-123
@@ -7,7 +7,8 @@
|
||||
//! 4. Model selection
|
||||
//! 5. Embeddings
|
||||
//! 6. Channel configuration
|
||||
//! 7. Heartbeat (background tasks)
|
||||
//! 7. Extensions (tool installation from registry)
|
||||
//! 8. Heartbeat (background tasks)
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
@@ -124,23 +125,43 @@ impl SetupWizard {
|
||||
}
|
||||
|
||||
/// Run the setup wizard.
|
||||
///
|
||||
/// Settings are persisted incrementally after each successful step so
|
||||
/// that progress is not lost if a later step fails. On re-run, existing
|
||||
/// settings are loaded from the database after Step 1 establishes a
|
||||
/// connection, so users don't have to re-enter everything.
|
||||
pub async fn run(&mut self) -> Result<(), SetupError> {
|
||||
print_header("IronClaw Setup Wizard");
|
||||
|
||||
if self.config.channels_only {
|
||||
// Channels-only mode: just step 6
|
||||
// Channels-only mode: reconnect to existing DB and load settings
|
||||
// before running the channel step, so secrets and save work.
|
||||
self.reconnect_existing_db().await?;
|
||||
print_step(1, 1, "Channel Configuration");
|
||||
self.step_channels().await?;
|
||||
} else {
|
||||
let total_steps = 7;
|
||||
let total_steps = 8;
|
||||
|
||||
// Step 1: Database
|
||||
print_step(1, total_steps, "Database Connection");
|
||||
self.step_database().await?;
|
||||
|
||||
// After establishing a DB connection, load any previously saved
|
||||
// settings so we recover progress from prior partial runs.
|
||||
// We must load BEFORE persisting, otherwise persist_after_step()
|
||||
// would overwrite prior settings with defaults.
|
||||
// Save Step 1 choices first so they aren't clobbered by stale
|
||||
// DB values (merge_from only applies non-default fields).
|
||||
let step1_settings = self.settings.clone();
|
||||
self.try_load_existing_settings().await;
|
||||
self.settings.merge_from(&step1_settings);
|
||||
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 2: Security
|
||||
print_step(2, total_steps, "Security");
|
||||
self.step_security().await?;
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 3: Inference provider selection (unless skipped)
|
||||
if !self.config.skip_auth {
|
||||
@@ -149,22 +170,31 @@ impl SetupWizard {
|
||||
} else {
|
||||
print_info("Skipping inference provider setup (using existing config)");
|
||||
}
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 4: Model selection
|
||||
print_step(4, total_steps, "Model Selection");
|
||||
self.step_model_selection().await?;
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 5: Embeddings
|
||||
print_step(5, total_steps, "Embeddings (Semantic Search)");
|
||||
self.step_embeddings()?;
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 6: Channel configuration
|
||||
print_step(6, total_steps, "Channel Configuration");
|
||||
self.step_channels().await?;
|
||||
self.persist_after_step().await;
|
||||
|
||||
// Step 7: Heartbeat
|
||||
print_step(7, total_steps, "Background Tasks");
|
||||
// Step 7: Extensions (tools)
|
||||
print_step(7, total_steps, "Extensions");
|
||||
self.step_extensions().await?;
|
||||
|
||||
// Step 8: Heartbeat
|
||||
print_step(8, total_steps, "Background Tasks");
|
||||
self.step_heartbeat()?;
|
||||
self.persist_after_step().await;
|
||||
}
|
||||
|
||||
// Save settings and print summary
|
||||
@@ -173,6 +203,99 @@ impl SetupWizard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reconnect to the existing database and load settings.
|
||||
///
|
||||
/// Used by channels-only mode (and future single-step modes) so that
|
||||
/// `init_secrets_context()` and `save_and_summarize()` have a live
|
||||
/// database connection and the wizard's `self.settings` reflects the
|
||||
/// previously saved configuration.
|
||||
async fn reconnect_existing_db(&mut self) -> Result<(), SetupError> {
|
||||
// Determine backend from env (set by bootstrap .env loaded in main).
|
||||
let backend = std::env::var("DATABASE_BACKEND").unwrap_or_else(|_| "postgres".to_string());
|
||||
|
||||
// Try libsql first if that's the configured backend.
|
||||
#[cfg(feature = "libsql")]
|
||||
if backend == "libsql" || backend == "turso" || backend == "sqlite" {
|
||||
return self.reconnect_libsql().await;
|
||||
}
|
||||
|
||||
// Try postgres (either explicitly configured or as default).
|
||||
#[cfg(feature = "postgres")]
|
||||
{
|
||||
let _ = &backend;
|
||||
return self.reconnect_postgres().await;
|
||||
}
|
||||
|
||||
#[allow(unreachable_code)]
|
||||
Err(SetupError::Database(
|
||||
"No database configured. Run full setup first (ironclaw onboard).".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Reconnect to an existing PostgreSQL database and load settings.
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn reconnect_postgres(&mut self) -> Result<(), SetupError> {
|
||||
let url = std::env::var("DATABASE_URL").map_err(|_| {
|
||||
SetupError::Database(
|
||||
"DATABASE_URL not set. Run full setup first (ironclaw onboard).".to_string(),
|
||||
)
|
||||
})?;
|
||||
|
||||
self.test_database_connection_postgres(&url).await?;
|
||||
self.settings.database_backend = Some("postgres".to_string());
|
||||
self.settings.database_url = Some(url.clone());
|
||||
|
||||
// Load existing settings from DB, then restore connection fields that
|
||||
// may not be persisted in the settings map.
|
||||
if let Some(ref pool) = self.db_pool {
|
||||
let store = crate::history::Store::from_pool(pool.clone());
|
||||
if let Ok(map) = store.get_all_settings("default").await {
|
||||
self.settings = Settings::from_db_map(&map);
|
||||
self.settings.database_backend = Some("postgres".to_string());
|
||||
self.settings.database_url = Some(url);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reconnect to an existing libSQL database and load settings.
|
||||
#[cfg(feature = "libsql")]
|
||||
async fn reconnect_libsql(&mut self) -> Result<(), SetupError> {
|
||||
let path = std::env::var("LIBSQL_PATH").unwrap_or_else(|_| {
|
||||
crate::config::default_libsql_path()
|
||||
.to_string_lossy()
|
||||
.to_string()
|
||||
});
|
||||
let turso_url = std::env::var("LIBSQL_URL").ok();
|
||||
let turso_token = std::env::var("LIBSQL_AUTH_TOKEN").ok();
|
||||
|
||||
self.test_database_connection_libsql(&path, turso_url.as_deref(), turso_token.as_deref())
|
||||
.await?;
|
||||
|
||||
self.settings.database_backend = Some("libsql".to_string());
|
||||
self.settings.libsql_path = Some(path.clone());
|
||||
if let Some(ref url) = turso_url {
|
||||
self.settings.libsql_url = Some(url.clone());
|
||||
}
|
||||
|
||||
// Load existing settings from DB, then restore connection fields that
|
||||
// may not be persisted in the settings map.
|
||||
if let Some(ref db) = self.db_backend {
|
||||
use crate::db::SettingsStore as _;
|
||||
if let Ok(map) = db.get_all_settings("default").await {
|
||||
self.settings = Settings::from_db_map(&map);
|
||||
self.settings.database_backend = Some("libsql".to_string());
|
||||
self.settings.libsql_path = Some(path);
|
||||
if let Some(url) = turso_url {
|
||||
self.settings.libsql_url = Some(url);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Step 1: Database connection.
|
||||
async fn step_database(&mut self) -> Result<(), SetupError> {
|
||||
// When both features are compiled, let the user choose.
|
||||
@@ -702,6 +825,20 @@ impl SetupWizard {
|
||||
.map_err(|e| SetupError::Auth(e.to_string()))?;
|
||||
|
||||
self.session_manager = Some(session);
|
||||
|
||||
// If the user chose the API key path, NEARAI_API_KEY is now set
|
||||
// in the environment. Persist it to the encrypted secrets store
|
||||
// so inject_llm_keys_from_secrets() can load it on future runs.
|
||||
if let Ok(api_key) = std::env::var("NEARAI_API_KEY")
|
||||
&& !api_key.is_empty()
|
||||
&& let Ok(ctx) = self.init_secrets_context().await
|
||||
{
|
||||
let key = SecretString::from(api_key);
|
||||
if let Err(e) = ctx.save_secret("llm_nearai_api_key", &key).await {
|
||||
tracing::warn!("Failed to persist NEARAI_API_KEY to secrets: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
print_success("NEAR AI configured");
|
||||
Ok(())
|
||||
}
|
||||
@@ -949,6 +1086,11 @@ impl SetupWizard {
|
||||
"anthropic::claude-sonnet-4-20250514".into(),
|
||||
"Claude Sonnet 4 (best quality)".into(),
|
||||
),
|
||||
(
|
||||
"openai::gpt-5.3-codex".into(),
|
||||
"GPT-5.3 Codex (flagship)".into(),
|
||||
),
|
||||
("openai::gpt-5.2".into(), "GPT-5.2".into()),
|
||||
("openai::gpt-4o".into(), "GPT-4o".into()),
|
||||
];
|
||||
|
||||
@@ -1279,7 +1421,9 @@ impl SetupWizard {
|
||||
.iter()
|
||||
.map(|(name, _)| name.clone())
|
||||
.collect();
|
||||
let wasm_channel_names = wasm_channel_option_names(&discovered_channels);
|
||||
|
||||
// Build channel list from registry (if available) + bundled + discovered
|
||||
let wasm_channel_names = build_channel_options(&discovered_channels);
|
||||
|
||||
// Build options list dynamically
|
||||
let mut options: Vec<(String, bool)> = vec![
|
||||
@@ -1290,11 +1434,15 @@ impl SetupWizard {
|
||||
),
|
||||
];
|
||||
|
||||
// Add available WASM channels (installed + bundled)
|
||||
// Add available WASM channels (installed + bundled + registry)
|
||||
for name in &wasm_channel_names {
|
||||
let is_enabled = self.settings.channels.wasm_channels.contains(name);
|
||||
let display_name = format!("{} (WASM)", capitalize_first(name));
|
||||
options.push((display_name, is_enabled));
|
||||
let label = if installed_names.contains(name) {
|
||||
format!("{} (installed)", capitalize_first(name))
|
||||
} else {
|
||||
format!("{} (will install)", capitalize_first(name))
|
||||
};
|
||||
options.push((label, is_enabled));
|
||||
}
|
||||
|
||||
let options_refs: Vec<(&str, bool)> =
|
||||
@@ -1315,6 +1463,10 @@ impl SetupWizard {
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Install selected channels that aren't already on disk
|
||||
let mut any_installed = false;
|
||||
|
||||
// Try bundled channels first (pre-compiled artifacts from channels-src/)
|
||||
if let Some(installed) = install_selected_bundled_channels(
|
||||
&channels_dir,
|
||||
&selected_wasm_channels,
|
||||
@@ -1323,7 +1475,31 @@ impl SetupWizard {
|
||||
.await?
|
||||
&& !installed.is_empty()
|
||||
{
|
||||
print_success(&format!("Installed channels: {}", installed.join(", ")));
|
||||
print_success(&format!(
|
||||
"Installed bundled channels: {}",
|
||||
installed.join(", ")
|
||||
));
|
||||
any_installed = true;
|
||||
}
|
||||
|
||||
// Then try registry channels (build from source for any still missing)
|
||||
let installed_from_registry = install_selected_registry_channels(
|
||||
&channels_dir,
|
||||
&selected_wasm_channels,
|
||||
&installed_names,
|
||||
)
|
||||
.await;
|
||||
|
||||
if !installed_from_registry.is_empty() {
|
||||
print_success(&format!(
|
||||
"Built from registry: {}",
|
||||
installed_from_registry.join(", ")
|
||||
));
|
||||
any_installed = true;
|
||||
}
|
||||
|
||||
// Re-discover after installs
|
||||
if any_installed {
|
||||
discovered_channels = discover_wasm_channels(&channels_dir).await;
|
||||
}
|
||||
|
||||
@@ -1414,7 +1590,134 @@ impl SetupWizard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Step 7: Heartbeat configuration.
|
||||
/// Step 7: Extensions (tools) installation from registry.
|
||||
async fn step_extensions(&mut self) -> Result<(), SetupError> {
|
||||
let catalog = match load_registry_catalog() {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
print_info("Extension registry not found. Skipping tool installation.");
|
||||
print_info("Install tools manually with: ironclaw tool install <path>");
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
|
||||
let tools: Vec<_> = catalog
|
||||
.list(Some(crate::registry::manifest::ManifestKind::Tool), None)
|
||||
.into_iter()
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if tools.is_empty() {
|
||||
print_info("No tools found in registry.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
print_info("Available tools from the extension registry:");
|
||||
print_info("Select which tools to install. You can install more later with:");
|
||||
print_info(" ironclaw registry install <name>");
|
||||
println!();
|
||||
|
||||
// Check which tools are already installed
|
||||
let tools_dir = dirs::home_dir()
|
||||
.ok_or_else(|| SetupError::Config("Could not determine home directory".into()))?
|
||||
.join(".ironclaw/tools");
|
||||
|
||||
let installed_tools = discover_installed_tools(&tools_dir).await;
|
||||
|
||||
// Build options: show display_name + description, pre-check "default" tagged + already installed
|
||||
let mut options: Vec<(String, bool)> = Vec::new();
|
||||
for tool in &tools {
|
||||
let is_installed = installed_tools.contains(&tool.name);
|
||||
let is_default = tool.tags.contains(&"default".to_string());
|
||||
let status = if is_installed { " (installed)" } else { "" };
|
||||
let auth_hint = tool
|
||||
.auth_summary
|
||||
.as_ref()
|
||||
.and_then(|a| a.method.as_deref())
|
||||
.map(|m| format!(" [{}]", m))
|
||||
.unwrap_or_default();
|
||||
|
||||
let label = format!(
|
||||
"{}{}{} - {}",
|
||||
tool.display_name, auth_hint, status, tool.description
|
||||
);
|
||||
options.push((label, is_default || is_installed));
|
||||
}
|
||||
|
||||
let options_refs: Vec<(&str, bool)> =
|
||||
options.iter().map(|(s, b)| (s.as_str(), *b)).collect();
|
||||
|
||||
let selected = select_many("Which tools do you want to install?", &options_refs)
|
||||
.map_err(SetupError::Io)?;
|
||||
|
||||
if selected.is_empty() {
|
||||
print_info("No tools selected.");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Install selected tools that aren't already on disk
|
||||
let repo_root = catalog.root().parent().unwrap_or(catalog.root());
|
||||
let installer = crate::registry::installer::RegistryInstaller::new(
|
||||
repo_root.to_path_buf(),
|
||||
tools_dir.clone(),
|
||||
dirs::home_dir()
|
||||
.unwrap_or_default()
|
||||
.join(".ironclaw/channels"),
|
||||
);
|
||||
|
||||
let mut installed_count = 0;
|
||||
let mut auth_needed: Vec<String> = Vec::new();
|
||||
|
||||
for idx in &selected {
|
||||
let tool = &tools[*idx];
|
||||
if installed_tools.contains(&tool.name) {
|
||||
continue; // Already installed, skip
|
||||
}
|
||||
|
||||
match installer.install_from_source(tool, false).await {
|
||||
Ok(outcome) => {
|
||||
print_success(&format!("Installed {}", outcome.name));
|
||||
installed_count += 1;
|
||||
|
||||
// Track auth needs
|
||||
if let Some(auth) = &tool.auth_summary
|
||||
&& auth.method.as_deref() != Some("none")
|
||||
&& auth.method.is_some()
|
||||
{
|
||||
let provider = auth.provider.as_deref().unwrap_or(&tool.name);
|
||||
// Only mention unique providers (Google tools share auth)
|
||||
let hint = format!(" {} - ironclaw tool auth {}", provider, tool.name);
|
||||
if !auth_needed
|
||||
.iter()
|
||||
.any(|h| h.starts_with(&format!(" {} -", provider)))
|
||||
{
|
||||
auth_needed.push(hint);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
print_error(&format!("Failed to install {}: {}", tool.display_name, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if installed_count > 0 {
|
||||
println!();
|
||||
print_success(&format!("{} tool(s) installed.", installed_count));
|
||||
}
|
||||
|
||||
if !auth_needed.is_empty() {
|
||||
println!();
|
||||
print_info("Some tools need authentication. Run after setup:");
|
||||
for hint in &auth_needed {
|
||||
print_info(hint);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Step 8: Heartbeat configuration.
|
||||
fn step_heartbeat(&mut self) -> Result<(), SetupError> {
|
||||
print_info("Heartbeat runs periodic background tasks (e.g., checking your calendar,");
|
||||
print_info("monitoring for notifications, running scheduled workflows).");
|
||||
@@ -1453,110 +1756,211 @@ impl SetupWizard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist current settings to the database.
|
||||
///
|
||||
/// Returns `Ok(true)` if settings were saved, `Ok(false)` if no database
|
||||
/// connection is available yet (e.g., before Step 1 completes).
|
||||
async fn persist_settings(&self) -> Result<bool, SetupError> {
|
||||
let db_map = self.settings.to_db_map();
|
||||
let saved = false;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
let saved = if !saved {
|
||||
if let Some(ref pool) = self.db_pool {
|
||||
let store = crate::history::Store::from_pool(pool.clone());
|
||||
store
|
||||
.set_all_settings("default", &db_map)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
SetupError::Database(format!("Failed to save settings to database: {}", e))
|
||||
})?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
saved
|
||||
};
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
let saved = if !saved {
|
||||
if let Some(ref backend) = self.db_backend {
|
||||
use crate::db::SettingsStore as _;
|
||||
backend
|
||||
.set_all_settings("default", &db_map)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
SetupError::Database(format!("Failed to save settings to database: {}", e))
|
||||
})?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
saved
|
||||
};
|
||||
|
||||
Ok(saved)
|
||||
}
|
||||
|
||||
/// Write bootstrap environment variables to `~/.ironclaw/.env`.
|
||||
///
|
||||
/// These are the chicken-and-egg settings needed before the database is
|
||||
/// connected (DATABASE_BACKEND, DATABASE_URL, LLM_BACKEND, etc.).
|
||||
fn write_bootstrap_env(&self) -> Result<(), SetupError> {
|
||||
let mut env_vars: Vec<(&str, String)> = Vec::new();
|
||||
|
||||
if let Some(ref backend) = self.settings.database_backend {
|
||||
env_vars.push(("DATABASE_BACKEND", backend.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.database_url {
|
||||
env_vars.push(("DATABASE_URL", url.clone()));
|
||||
}
|
||||
if let Some(ref path) = self.settings.libsql_path {
|
||||
env_vars.push(("LIBSQL_PATH", path.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.libsql_url {
|
||||
env_vars.push(("LIBSQL_URL", url.clone()));
|
||||
}
|
||||
|
||||
// LLM bootstrap vars: same chicken-and-egg problem as DATABASE_BACKEND.
|
||||
// Config::from_env() needs the backend before the DB is connected.
|
||||
if let Some(ref backend) = self.settings.llm_backend {
|
||||
env_vars.push(("LLM_BACKEND", backend.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.openai_compatible_base_url {
|
||||
env_vars.push(("LLM_BASE_URL", url.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.ollama_base_url {
|
||||
env_vars.push(("OLLAMA_BASE_URL", url.clone()));
|
||||
}
|
||||
|
||||
// Preserve NEARAI_API_KEY if present (set by API key auth flow)
|
||||
if let Ok(api_key) = std::env::var("NEARAI_API_KEY")
|
||||
&& !api_key.is_empty()
|
||||
{
|
||||
env_vars.push(("NEARAI_API_KEY", api_key));
|
||||
}
|
||||
|
||||
// Always write ONBOARD_COMPLETED so that check_onboard_needed()
|
||||
// (which runs before the DB is connected) knows to skip re-onboarding.
|
||||
if self.settings.onboard_completed {
|
||||
env_vars.push(("ONBOARD_COMPLETED", "true".to_string()));
|
||||
}
|
||||
|
||||
if !env_vars.is_empty() {
|
||||
let pairs: Vec<(&str, &str)> = env_vars.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
crate::bootstrap::save_bootstrap_env(&pairs).map_err(|e| {
|
||||
SetupError::Io(std::io::Error::other(format!(
|
||||
"Failed to save bootstrap env to .env: {}",
|
||||
e
|
||||
)))
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Persist settings to DB and bootstrap .env after each step.
|
||||
///
|
||||
/// Silently ignores errors (e.g., DB not connected yet before step 1
|
||||
/// completes). This is best-effort incremental persistence.
|
||||
async fn persist_after_step(&self) {
|
||||
// Write bootstrap .env (always possible)
|
||||
if let Err(e) = self.write_bootstrap_env() {
|
||||
tracing::debug!("Could not write bootstrap env after step: {}", e);
|
||||
}
|
||||
|
||||
// Persist to DB
|
||||
match self.persist_settings().await {
|
||||
Ok(true) => tracing::debug!("Settings persisted to database after step"),
|
||||
Ok(false) => tracing::debug!("No DB connection yet, skipping settings persist"),
|
||||
Err(e) => tracing::debug!("Could not persist settings after step: {}", e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Load previously saved settings from the database after Step 1
|
||||
/// establishes a connection.
|
||||
///
|
||||
/// This enables recovery from partial onboarding runs: if the user
|
||||
/// completed steps 1-4 previously but step 5 failed, re-running
|
||||
/// the wizard will pre-populate settings from the database.
|
||||
///
|
||||
/// **Callers must re-apply any wizard choices made before this call**
|
||||
/// via `self.settings.merge_from(&step_settings)`, since `merge_from`
|
||||
/// prefers the `other` argument's non-default values. Without this,
|
||||
/// stale DB values would overwrite fresh user choices.
|
||||
async fn try_load_existing_settings(&mut self) {
|
||||
let loaded = false;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
let loaded = if !loaded {
|
||||
if let Some(ref pool) = self.db_pool {
|
||||
let store = crate::history::Store::from_pool(pool.clone());
|
||||
match store.get_all_settings("default").await {
|
||||
Ok(db_map) if !db_map.is_empty() => {
|
||||
let existing = Settings::from_db_map(&db_map);
|
||||
self.settings.merge_from(&existing);
|
||||
tracing::info!("Loaded {} existing settings from database", db_map.len());
|
||||
true
|
||||
}
|
||||
Ok(_) => false,
|
||||
Err(e) => {
|
||||
tracing::debug!("Could not load existing settings: {}", e);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
loaded
|
||||
};
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
let loaded = if !loaded {
|
||||
if let Some(ref backend) = self.db_backend {
|
||||
use crate::db::SettingsStore as _;
|
||||
match backend.get_all_settings("default").await {
|
||||
Ok(db_map) if !db_map.is_empty() => {
|
||||
let existing = Settings::from_db_map(&db_map);
|
||||
self.settings.merge_from(&existing);
|
||||
tracing::info!("Loaded {} existing settings from database", db_map.len());
|
||||
true
|
||||
}
|
||||
Ok(_) => false,
|
||||
Err(e) => {
|
||||
tracing::debug!("Could not load existing settings: {}", e);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
loaded
|
||||
};
|
||||
|
||||
// Suppress unused variable warning when only one backend is compiled.
|
||||
let _ = loaded;
|
||||
}
|
||||
|
||||
/// Save settings to the database and `~/.ironclaw/.env`, then print summary.
|
||||
async fn save_and_summarize(&mut self) -> Result<(), SetupError> {
|
||||
self.settings.onboard_completed = true;
|
||||
|
||||
// Write all settings to the database (whichever backend is active).
|
||||
{
|
||||
let db_map = self.settings.to_db_map();
|
||||
let saved = false;
|
||||
// Final persist (idempotent — earlier incremental saves already wrote
|
||||
// most settings, but this ensures onboard_completed is saved).
|
||||
let saved = self.persist_settings().await?;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
let saved = if !saved {
|
||||
if let Some(ref pool) = self.db_pool {
|
||||
let store = crate::history::Store::from_pool(pool.clone());
|
||||
store
|
||||
.set_all_settings("default", &db_map)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
SetupError::Database(format!(
|
||||
"Failed to save settings to database: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
saved
|
||||
};
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
let saved = if !saved {
|
||||
if let Some(ref backend) = self.db_backend {
|
||||
use crate::db::SettingsStore as _;
|
||||
backend
|
||||
.set_all_settings("default", &db_map)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
SetupError::Database(format!(
|
||||
"Failed to save settings to database: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else {
|
||||
saved
|
||||
};
|
||||
|
||||
if !saved {
|
||||
return Err(SetupError::Database(
|
||||
"No database connection, cannot save settings".to_string(),
|
||||
));
|
||||
}
|
||||
if !saved {
|
||||
return Err(SetupError::Database(
|
||||
"No database connection, cannot save settings".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Persist database bootstrap vars to ~/.ironclaw/.env.
|
||||
// These are the chicken-and-egg settings: we need them to decide
|
||||
// which database to connect to, so they can't live in the database.
|
||||
{
|
||||
let mut env_vars: Vec<(&str, String)> = Vec::new();
|
||||
|
||||
if let Some(ref backend) = self.settings.database_backend {
|
||||
env_vars.push(("DATABASE_BACKEND", backend.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.database_url {
|
||||
env_vars.push(("DATABASE_URL", url.clone()));
|
||||
}
|
||||
if let Some(ref path) = self.settings.libsql_path {
|
||||
env_vars.push(("LIBSQL_PATH", path.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.libsql_url {
|
||||
env_vars.push(("LIBSQL_URL", url.clone()));
|
||||
}
|
||||
|
||||
// LLM bootstrap vars: same chicken-and-egg problem as DATABASE_BACKEND.
|
||||
// Config::from_env() needs the backend before the DB is connected.
|
||||
if let Some(ref backend) = self.settings.llm_backend {
|
||||
env_vars.push(("LLM_BACKEND", backend.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.openai_compatible_base_url {
|
||||
env_vars.push(("LLM_BASE_URL", url.clone()));
|
||||
}
|
||||
if let Some(ref url) = self.settings.ollama_base_url {
|
||||
env_vars.push(("OLLAMA_BASE_URL", url.clone()));
|
||||
}
|
||||
|
||||
// Always write ONBOARD_COMPLETED so that check_onboard_needed()
|
||||
// (which runs before the DB is connected) knows to skip re-onboarding.
|
||||
env_vars.push(("ONBOARD_COMPLETED", "true".to_string()));
|
||||
|
||||
if !env_vars.is_empty() {
|
||||
let pairs: Vec<(&str, &str)> =
|
||||
env_vars.iter().map(|(k, v)| (*k, v.as_str())).collect();
|
||||
crate::bootstrap::save_bootstrap_env(&pairs).map_err(|e| {
|
||||
SetupError::Io(std::io::Error::other(format!(
|
||||
"Failed to save bootstrap env to .env: {}",
|
||||
e
|
||||
)))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
// Write bootstrap env (also idempotent)
|
||||
self.write_bootstrap_env()?;
|
||||
|
||||
println!();
|
||||
print_success("Configuration saved to database");
|
||||
@@ -1714,12 +2118,14 @@ fn mask_password_in_url(url: &str) -> String {
|
||||
/// Returns `(model_id, display_label)` pairs. Falls back to static defaults on error.
|
||||
async fn fetch_anthropic_models(cached_key: Option<&str>) -> Vec<(String, String)> {
|
||||
let static_defaults = vec![
|
||||
("claude-sonnet-4-20250514".into(), "Claude Sonnet 4".into()),
|
||||
("claude-opus-4-20250514".into(), "Claude Opus 4".into()),
|
||||
(
|
||||
"claude-3-5-haiku-20241022".into(),
|
||||
"Claude 3.5 Haiku (fast)".into(),
|
||||
"claude-opus-4-6".into(),
|
||||
"Claude Opus 4.6 (latest flagship)".into(),
|
||||
),
|
||||
("claude-sonnet-4-6".into(), "Claude Sonnet 4.6".into()),
|
||||
("claude-opus-4-5".into(), "Claude Opus 4.5".into()),
|
||||
("claude-sonnet-4-5".into(), "Claude Sonnet 4.5".into()),
|
||||
("claude-haiku-4-5".into(), "Claude Haiku 4.5 (fast)".into()),
|
||||
];
|
||||
|
||||
let api_key = cached_key
|
||||
@@ -1780,10 +2186,21 @@ async fn fetch_anthropic_models(cached_key: Option<&str>) -> Vec<(String, String
|
||||
/// Returns `(model_id, display_label)` pairs. Falls back to static defaults on error.
|
||||
async fn fetch_openai_models(cached_key: Option<&str>) -> Vec<(String, String)> {
|
||||
let static_defaults = vec![
|
||||
("gpt-5".into(), "GPT-5 (flagship)".into()),
|
||||
("gpt-5-mini".into(), "GPT-5 Mini (fast)".into()),
|
||||
(
|
||||
"gpt-5.3-codex".into(),
|
||||
"GPT-5.3 Codex (latest flagship)".into(),
|
||||
),
|
||||
("gpt-5.2-codex".into(), "GPT-5.2 Codex".into()),
|
||||
("gpt-5.2".into(), "GPT-5.2".into()),
|
||||
(
|
||||
"gpt-5.1-codex-mini".into(),
|
||||
"GPT-5.1 Codex Mini (fast)".into(),
|
||||
),
|
||||
("gpt-5".into(), "GPT-5".into()),
|
||||
("gpt-5-mini".into(), "GPT-5 Mini".into()),
|
||||
("gpt-4.1".into(), "GPT-4.1".into()),
|
||||
("gpt-4o".into(), "GPT-4o".into()),
|
||||
("gpt-4.1-mini".into(), "GPT-4.1 Mini".into()),
|
||||
("o4-mini".into(), "o4-mini (fast reasoning)".into()),
|
||||
("o3".into(), "o3 (reasoning)".into()),
|
||||
];
|
||||
|
||||
@@ -1864,11 +2281,15 @@ fn openai_model_priority(model_id: &str) -> usize {
|
||||
let id = model_id.to_ascii_lowercase();
|
||||
|
||||
const EXACT_PRIORITY: &[&str] = &[
|
||||
"gpt-5.3-codex",
|
||||
"gpt-5.2-codex",
|
||||
"gpt-5.2",
|
||||
"gpt-5.1-codex-mini",
|
||||
"gpt-5",
|
||||
"gpt-5-mini",
|
||||
"gpt-5-nano",
|
||||
"o3",
|
||||
"o4-mini",
|
||||
"o3",
|
||||
"o1",
|
||||
"gpt-4.1",
|
||||
"gpt-4.1-mini",
|
||||
@@ -1880,7 +2301,7 @@ fn openai_model_priority(model_id: &str) -> usize {
|
||||
}
|
||||
|
||||
const PREFIX_PRIORITY: &[&str] = &[
|
||||
"gpt-5-", "o3-", "o4-", "o1-", "gpt-4.1-", "gpt-4o-", "gpt-3.5-", "chatgpt-",
|
||||
"gpt-5.", "gpt-5-", "o3-", "o4-", "o1-", "gpt-4.1-", "gpt-4o-", "gpt-3.5-", "chatgpt-",
|
||||
];
|
||||
if let Some(pos) = PREFIX_PRIORITY
|
||||
.iter()
|
||||
@@ -2065,15 +2486,161 @@ async fn install_missing_bundled_channels(
|
||||
Ok(installed)
|
||||
}
|
||||
|
||||
fn wasm_channel_option_names(discovered: &[(String, ChannelCapabilitiesFile)]) -> Vec<String> {
|
||||
/// Build channel options from discovered channels + bundled + registry catalog.
|
||||
///
|
||||
/// Returns a deduplicated, sorted list of channel names available for selection.
|
||||
fn build_channel_options(discovered: &[(String, ChannelCapabilitiesFile)]) -> Vec<String> {
|
||||
let mut names: Vec<String> = discovered.iter().map(|(name, _)| name.clone()).collect();
|
||||
|
||||
// Add bundled channels
|
||||
for bundled in available_channel_names().iter().copied() {
|
||||
if !names.iter().any(|name| name == bundled) {
|
||||
names.push(bundled.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
// Add registry channels
|
||||
if let Some(catalog) = load_registry_catalog() {
|
||||
for manifest in catalog.list(Some(crate::registry::manifest::ManifestKind::Channel), None) {
|
||||
if !names.iter().any(|n| n == &manifest.name) {
|
||||
names.push(manifest.name.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
/// Try to load the registry catalog. Returns None if the registry directory
|
||||
/// cannot be found (e.g. running from an installed binary without the repo).
|
||||
fn load_registry_catalog() -> Option<crate::registry::catalog::RegistryCatalog> {
|
||||
// Try relative to current directory (dev usage)
|
||||
let cwd = std::env::current_dir().ok()?;
|
||||
let candidate = cwd.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
|
||||
}
|
||||
|
||||
// Try relative to executable
|
||||
if let Ok(exe) = std::env::current_exe()
|
||||
&& let Some(parent) = exe.parent()
|
||||
{
|
||||
let candidate = parent.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
|
||||
}
|
||||
if let Some(grandparent) = parent.parent() {
|
||||
let candidate = grandparent.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let candidate = manifest_dir.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Install selected channels from the registry that aren't already on disk
|
||||
/// and weren't handled by the bundled installer.
|
||||
///
|
||||
/// This builds channels from source using `cargo component build`.
|
||||
async fn install_selected_registry_channels(
|
||||
channels_dir: &std::path::Path,
|
||||
selected_channels: &[String],
|
||||
already_installed: &HashSet<String>,
|
||||
) -> Vec<String> {
|
||||
let catalog = match load_registry_catalog() {
|
||||
Some(c) => c,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
|
||||
let repo_root = catalog
|
||||
.root()
|
||||
.parent()
|
||||
.unwrap_or(catalog.root())
|
||||
.to_path_buf();
|
||||
|
||||
let bundled: HashSet<&str> = available_channel_names().iter().copied().collect();
|
||||
let mut installed = Vec::new();
|
||||
|
||||
for name in selected_channels {
|
||||
// Skip if already installed or handled by bundled installer
|
||||
if already_installed.contains(name) || bundled.contains(name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if already on disk (may have been installed between bundled and here)
|
||||
let wasm_on_disk = channels_dir.join(format!("{}.wasm", name)).exists()
|
||||
|| channels_dir.join(format!("{}-channel.wasm", name)).exists();
|
||||
if wasm_on_disk {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Look up in registry
|
||||
let manifest = match catalog.get(&format!("channels/{}", name)) {
|
||||
Some(m) => m,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let installer = crate::registry::installer::RegistryInstaller::new(
|
||||
repo_root.clone(),
|
||||
dirs::home_dir().unwrap_or_default().join(".ironclaw/tools"),
|
||||
channels_dir.to_path_buf(),
|
||||
);
|
||||
|
||||
match installer.install_from_source(manifest, false).await {
|
||||
Ok(_) => {
|
||||
installed.push(name.clone());
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
channel = %name,
|
||||
error = %e,
|
||||
"Failed to install channel from registry"
|
||||
);
|
||||
crate::setup::prompts::print_error(&format!(
|
||||
"Failed to install channel '{}': {}",
|
||||
name, e
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
installed
|
||||
}
|
||||
|
||||
/// Discover which tools are already installed in the tools directory.
|
||||
///
|
||||
/// Returns a set of tool names (the stem of .wasm files).
|
||||
async fn discover_installed_tools(tools_dir: &std::path::Path) -> HashSet<String> {
|
||||
let mut names = HashSet::new();
|
||||
|
||||
if !tools_dir.is_dir() {
|
||||
return names;
|
||||
}
|
||||
|
||||
let mut entries = match tokio::fs::read_dir(tools_dir).await {
|
||||
Ok(e) => e,
|
||||
Err(_) => return names,
|
||||
};
|
||||
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) == Some("wasm")
|
||||
&& let Some(stem) = path.file_stem().and_then(|s| s.to_str())
|
||||
{
|
||||
names.insert(stem.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
names
|
||||
}
|
||||
|
||||
@@ -2187,9 +2754,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_channel_option_names_includes_available_when_missing() {
|
||||
fn test_build_channel_options_includes_available_when_missing() {
|
||||
let discovered = Vec::new();
|
||||
let options = wasm_channel_option_names(&discovered);
|
||||
let options = build_channel_options(&discovered);
|
||||
let available = available_channel_names();
|
||||
// All available (built) channels should appear
|
||||
for name in &available {
|
||||
@@ -2202,9 +2769,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wasm_channel_option_names_dedupes_available() {
|
||||
fn test_build_channel_options_dedupes_available() {
|
||||
let discovered = vec![(String::from("telegram"), ChannelCapabilitiesFile::default())];
|
||||
let options = wasm_channel_option_names(&discovered);
|
||||
let options = build_channel_options(&discovered);
|
||||
// telegram should appear exactly once despite being both discovered and available
|
||||
assert_eq!(
|
||||
options.iter().filter(|n| *n == "telegram").count(),
|
||||
@@ -2230,7 +2797,7 @@ mod tests {
|
||||
let _guard = EnvGuard::clear("OPENAI_API_KEY");
|
||||
let models = fetch_openai_models(None).await;
|
||||
assert!(!models.is_empty());
|
||||
assert_eq!(models[0].0, "gpt-5");
|
||||
assert_eq!(models[0].0, "gpt-5.3-codex");
|
||||
assert!(
|
||||
models.iter().any(|(id, _)| id.contains("gpt")),
|
||||
"static defaults should include a GPT model"
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# Tool System
|
||||
|
||||
## Adding a New Tool
|
||||
|
||||
### Built-in Tools (Rust)
|
||||
|
||||
1. Create `src/tools/builtin/my_tool.rs`
|
||||
2. Implement the `Tool` trait
|
||||
3. Add `mod my_tool;` and `pub use` in `src/tools/builtin/mod.rs`
|
||||
4. Register in `ToolRegistry::register_builtin_tools()` in `registry.rs`
|
||||
5. Add tests
|
||||
|
||||
### WASM Tools (Recommended)
|
||||
|
||||
WASM tools are the preferred way to add new capabilities. They run in a sandboxed environment with explicit capabilities.
|
||||
|
||||
1. Create a new crate in `tools-src/<name>/`
|
||||
2. Implement the WIT interface (`wit/tool.wit`)
|
||||
3. Create `<name>.capabilities.json` declaring required permissions
|
||||
4. Build with `cargo build --target wasm32-wasip2 --release`
|
||||
5. Install with `ironclaw tool install path/to/tool.wasm`
|
||||
|
||||
See `tools-src/` for examples.
|
||||
|
||||
## Tool Architecture Principles
|
||||
|
||||
**CRITICAL: Keep tool-specific logic out of the main agent codebase.**
|
||||
|
||||
The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through capabilities files.
|
||||
|
||||
### What Goes in Tools (capabilities.json)
|
||||
|
||||
- API endpoints the tool needs (HTTP allowlist)
|
||||
- Credentials required (secret names, injection locations)
|
||||
- Rate limits and timeouts
|
||||
- Auth setup instructions (see below)
|
||||
- Workspace paths the tool can read
|
||||
|
||||
### What Does NOT Go in Main Agent
|
||||
|
||||
- Service-specific auth flows (OAuth for Notion, Slack, etc.)
|
||||
- Service-specific CLI commands (`auth notion`, `auth slack`)
|
||||
- Service-specific configuration handling
|
||||
- Hardcoded API URLs or token formats
|
||||
|
||||
### Tool Authentication
|
||||
|
||||
Tools declare their auth requirements in `<tool>.capabilities.json` under the `auth` section. Two methods are supported:
|
||||
|
||||
#### OAuth (Browser-based login)
|
||||
|
||||
For services that support OAuth, users just click through browser login:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "notion_api_token",
|
||||
"display_name": "Notion",
|
||||
"oauth": {
|
||||
"authorization_url": "https://api.notion.com/v1/oauth/authorize",
|
||||
"token_url": "https://api.notion.com/v1/oauth/token",
|
||||
"client_id_env": "NOTION_OAUTH_CLIENT_ID",
|
||||
"client_secret_env": "NOTION_OAUTH_CLIENT_SECRET",
|
||||
"scopes": [],
|
||||
"use_pkce": false,
|
||||
"extra_params": { "owner": "user" }
|
||||
},
|
||||
"env_var": "NOTION_TOKEN"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To enable OAuth for a tool:
|
||||
1. Register a public OAuth app with the service (e.g., notion.so/my-integrations)
|
||||
2. Configure redirect URIs: `http://localhost:9876/callback` through `http://localhost:9886/callback`
|
||||
3. Set environment variables for client_id and client_secret
|
||||
|
||||
#### Manual Token Entry (Fallback)
|
||||
|
||||
For services without OAuth or when OAuth isn't configured:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "openai_api_key",
|
||||
"display_name": "OpenAI",
|
||||
"instructions": "Get your API key from platform.openai.com/api-keys",
|
||||
"setup_url": "https://platform.openai.com/api-keys",
|
||||
"token_hint": "Starts with 'sk-'",
|
||||
"env_var": "OPENAI_API_KEY"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Auth Flow Priority
|
||||
|
||||
When running `ironclaw tool auth <tool>`:
|
||||
|
||||
1. Check `env_var` - if set in environment, use it directly
|
||||
2. Check `oauth` - if configured, open browser for OAuth flow
|
||||
3. Fall back to `instructions` + manual token entry
|
||||
|
||||
The agent reads auth config from the tool's capabilities file and provides the appropriate flow. No service-specific code in the main agent.
|
||||
|
||||
### WASM Tools vs MCP Servers: When to Use Which
|
||||
|
||||
Both are first-class in the extension system (`ironclaw tool install` handles both), but they have different strengths.
|
||||
|
||||
**WASM Tools (IronClaw native)**
|
||||
|
||||
- Sandboxed: fuel metering, memory limits, no access except what's allowlisted
|
||||
- Credentials injected by host runtime, tool code never sees the actual token
|
||||
- Output scanned for secret leakage before returning to the LLM
|
||||
- Auth (OAuth/manual) declared in `capabilities.json`, agent handles the flow
|
||||
- Single binary, no process management, works offline
|
||||
- Cost: must build yourself in Rust, no ecosystem, synchronous only
|
||||
|
||||
**MCP Servers (Model Context Protocol)**
|
||||
|
||||
- Growing ecosystem of pre-built servers (GitHub, Notion, Postgres, etc.)
|
||||
- Any language (TypeScript/Python most common)
|
||||
- Can do websockets, streaming, background polling
|
||||
- Cost: external process with full system access (no sandbox), manages own credentials, IronClaw can't prevent leaks
|
||||
|
||||
**Decision guide:**
|
||||
|
||||
| Scenario | Use |
|
||||
|----------|-----|
|
||||
| Good MCP server already exists | **MCP** |
|
||||
| Handles sensitive credentials (email send, banking) | **WASM** |
|
||||
| Quick prototype or one-off integration | **MCP** |
|
||||
| Core capability you'll maintain long-term | **WASM** |
|
||||
| Needs background connections (websockets, polling) | **MCP** |
|
||||
| Multiple tools share one OAuth token (e.g., Google suite) | **WASM** |
|
||||
|
||||
The LLM-facing interface is identical for both (tool name, schema, execute), so swapping between them is transparent to the agent.
|
||||
@@ -189,8 +189,8 @@ impl Tool for HttpTool {
|
||||
}
|
||||
},
|
||||
"body": {
|
||||
"type": "string",
|
||||
"description": "Request body. Use plain text or serialized JSON."
|
||||
"type": ["object", "array", "string", "number", "boolean", "null"],
|
||||
"description": "Request body (for POST/PUT/PATCH)"
|
||||
},
|
||||
"timeout_secs": {
|
||||
"type": "integer",
|
||||
@@ -361,13 +361,6 @@ impl Tool for HttpTool {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_http_tool_schema_body_has_type() {
|
||||
let tool = HttpTool::new();
|
||||
let schema = tool.parameters_schema();
|
||||
assert_eq!(schema["properties"]["body"]["type"], "string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_tool_schema_headers_is_array() {
|
||||
let tool = HttpTool::new();
|
||||
@@ -460,4 +453,18 @@ mod tests {
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_tool_schema_body_has_type() {
|
||||
let schema = HttpTool::new().parameters_schema();
|
||||
let body = schema
|
||||
.get("properties")
|
||||
.and_then(|p| p.get("body"))
|
||||
.expect("body schema missing");
|
||||
|
||||
assert!(
|
||||
body.get("type").is_some(),
|
||||
"body schema must include a type for OpenAI-compatible tool validation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ impl Tool for JsonTool {
|
||||
"description": "The JSON operation to perform"
|
||||
},
|
||||
"data": {
|
||||
"type": "string",
|
||||
"description": "JSON input string. For query/stringify/validate, pass serialized JSON."
|
||||
"type": ["string", "object", "array", "number", "boolean", "null"],
|
||||
"description": "JSON input data. Pass a string for parse, any type otherwise."
|
||||
},
|
||||
"path": {
|
||||
"type": "string",
|
||||
@@ -154,13 +154,6 @@ fn query_json(data: &serde_json::Value, path: &str) -> Result<serde_json::Value,
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_json_tool_schema_data_has_type() {
|
||||
let tool = JsonTool;
|
||||
let schema = tool.parameters_schema();
|
||||
assert_eq!(schema["properties"]["data"]["type"], "string");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_json() {
|
||||
let data = serde_json::json!({
|
||||
@@ -197,4 +190,18 @@ mod tests {
|
||||
let err = parse_json_input(&input).unwrap_err();
|
||||
assert!(err.to_string().contains("invalid JSON input"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_tool_schema_data_has_type() {
|
||||
let schema = JsonTool.parameters_schema();
|
||||
let data = schema
|
||||
.get("properties")
|
||||
.and_then(|p| p.get("data"))
|
||||
.expect("data schema missing");
|
||||
|
||||
assert!(
|
||||
data.get("type").is_some(),
|
||||
"data schema must include a type for OpenAI-compatible tool validation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+58
-16
@@ -507,25 +507,47 @@ impl ShellTool {
|
||||
.spawn()
|
||||
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to spawn command: {}", e)))?;
|
||||
|
||||
// Wait with timeout
|
||||
// Drain stdout/stderr concurrently with wait() to prevent deadlocks.
|
||||
// If we call wait() without draining the pipes and the child's output
|
||||
// exceeds the OS pipe buffer (64KB Linux, 16KB macOS), the child blocks
|
||||
// on write and wait() never returns.
|
||||
let stdout_handle = child.stdout.take();
|
||||
let stderr_handle = child.stderr.take();
|
||||
|
||||
let result = tokio::time::timeout(timeout, async {
|
||||
let status = child.wait().await?;
|
||||
let stdout_fut = async {
|
||||
if let Some(mut out) = stdout_handle {
|
||||
let mut buf = Vec::new();
|
||||
(&mut out)
|
||||
.take(MAX_OUTPUT_SIZE as u64)
|
||||
.read_to_end(&mut buf)
|
||||
.await
|
||||
.ok();
|
||||
// Drain any remaining output so the child does not block
|
||||
tokio::io::copy(&mut out, &mut tokio::io::sink()).await.ok();
|
||||
String::from_utf8_lossy(&buf).to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Read stdout
|
||||
let mut stdout = String::new();
|
||||
if let Some(mut out) = child.stdout.take() {
|
||||
let mut buf = vec![0u8; MAX_OUTPUT_SIZE];
|
||||
let n = out.read(&mut buf).await.unwrap_or(0);
|
||||
stdout = String::from_utf8_lossy(&buf[..n]).to_string();
|
||||
}
|
||||
let stderr_fut = async {
|
||||
if let Some(mut err) = stderr_handle {
|
||||
let mut buf = Vec::new();
|
||||
(&mut err)
|
||||
.take(MAX_OUTPUT_SIZE as u64)
|
||||
.read_to_end(&mut buf)
|
||||
.await
|
||||
.ok();
|
||||
tokio::io::copy(&mut err, &mut tokio::io::sink()).await.ok();
|
||||
String::from_utf8_lossy(&buf).to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Read stderr
|
||||
let mut stderr = String::new();
|
||||
if let Some(mut err) = child.stderr.take() {
|
||||
let mut buf = vec![0u8; MAX_OUTPUT_SIZE];
|
||||
let n = err.read(&mut buf).await.unwrap_or(0);
|
||||
stderr = String::from_utf8_lossy(&buf[..n]).to_string();
|
||||
}
|
||||
let (stdout, stderr, wait_result) = tokio::join!(stdout_fut, stderr_fut, child.wait());
|
||||
let status = wait_result?;
|
||||
|
||||
// Combine output
|
||||
let output = if stderr.is_empty() {
|
||||
@@ -1184,6 +1206,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_large_output_command() {
|
||||
let tool = ShellTool::new().with_timeout(Duration::from_secs(10));
|
||||
let ctx = JobContext::default();
|
||||
|
||||
// Generate output larger than OS pipe buffer (64KB on Linux, 16KB on macOS).
|
||||
// Without draining pipes before wait(), this would deadlock.
|
||||
let result = tool
|
||||
.execute(
|
||||
serde_json::json!({"command": "python3 -c \"print('A' * 131072)\""}),
|
||||
&ctx,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let output = result.result.get("output").unwrap().as_str().unwrap();
|
||||
assert_eq!(output.len(), MAX_OUTPUT_SIZE);
|
||||
assert_eq!(result.result.get("exit_code").unwrap().as_i64().unwrap(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_netcat_blocked_at_execution() {
|
||||
let tool = ShellTool::new();
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# Workspace & Memory System
|
||||
|
||||
Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure.
|
||||
|
||||
## Key Principles
|
||||
|
||||
1. **"Memory is database, not RAM"** - If you want to remember something, write it explicitly
|
||||
2. **Flexible structure** - Create any directory/file hierarchy you need
|
||||
3. **Self-documenting** - Use README.md files to describe directory structure
|
||||
4. **Hybrid search** - Combines FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion
|
||||
|
||||
## Filesystem Structure
|
||||
|
||||
```
|
||||
workspace/
|
||||
├── README.md <- Root runbook/index
|
||||
├── MEMORY.md <- Long-term curated memory
|
||||
├── HEARTBEAT.md <- Periodic checklist
|
||||
├── IDENTITY.md <- Agent name, nature, vibe
|
||||
├── SOUL.md <- Core values
|
||||
├── AGENTS.md <- Behavior instructions
|
||||
├── USER.md <- User context
|
||||
├── context/ <- Identity-related docs
|
||||
│ ├── vision.md
|
||||
│ └── priorities.md
|
||||
├── daily/ <- Daily logs
|
||||
│ ├── 2024-01-15.md
|
||||
│ └── 2024-01-16.md
|
||||
├── projects/ <- Arbitrary structure
|
||||
│ └── alpha/
|
||||
│ ├── README.md
|
||||
│ └── notes.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
## Using the Workspace
|
||||
|
||||
```rust
|
||||
use crate::workspace::{Workspace, OpenAiEmbeddings, paths};
|
||||
|
||||
// Create workspace for a user
|
||||
let workspace = Workspace::new("user_123", pool)
|
||||
.with_embeddings(Arc::new(OpenAiEmbeddings::new(api_key)));
|
||||
|
||||
// Read/write any path
|
||||
let doc = workspace.read("projects/alpha/notes.md").await?;
|
||||
workspace.write("context/priorities.md", "# Priorities\n\n1. Feature X").await?;
|
||||
workspace.append("daily/2024-01-15.md", "Completed task X").await?;
|
||||
|
||||
// Convenience methods for well-known files
|
||||
workspace.append_memory("User prefers dark mode").await?;
|
||||
workspace.append_daily_log("Session note").await?;
|
||||
|
||||
// List directory contents
|
||||
let entries = workspace.list("projects/").await?;
|
||||
|
||||
// Search (hybrid FTS + vector)
|
||||
let results = workspace.search("dark mode preference", 5).await?;
|
||||
|
||||
// Get system prompt from identity files
|
||||
let prompt = workspace.system_prompt().await?;
|
||||
```
|
||||
|
||||
## Memory Tools
|
||||
|
||||
Four tools for LLM use:
|
||||
|
||||
- **`memory_search`** - Hybrid search, MUST be called before answering questions about prior work
|
||||
- **`memory_write`** - Write to any path (memory, daily_log, or custom paths)
|
||||
- **`memory_read`** - Read any file by path
|
||||
- **`memory_tree`** - View workspace structure as a tree (depth parameter, default 1)
|
||||
|
||||
## Hybrid Search (RRF)
|
||||
|
||||
Combines full-text search and vector similarity using Reciprocal Rank Fusion:
|
||||
|
||||
```
|
||||
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||
```
|
||||
|
||||
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
||||
|
||||
**Backend differences:**
|
||||
- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF
|
||||
- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired)
|
||||
|
||||
## Heartbeat System
|
||||
|
||||
Proactive periodic execution (default: 30 minutes):
|
||||
|
||||
1. Reads `HEARTBEAT.md` checklist
|
||||
2. Runs agent turn with checklist prompt
|
||||
3. If findings, notifies via channel
|
||||
4. If nothing, agent replies "HEARTBEAT_OK" (no notification)
|
||||
|
||||
```rust
|
||||
use crate::agent::{HeartbeatConfig, spawn_heartbeat};
|
||||
|
||||
let config = HeartbeatConfig::default()
|
||||
.with_interval(Duration::from_secs(60 * 30))
|
||||
.with_notify("user_123", "telegram");
|
||||
|
||||
spawn_heartbeat(config, workspace, llm, response_tx);
|
||||
```
|
||||
|
||||
## Chunking Strategy
|
||||
|
||||
Documents are chunked for search indexing:
|
||||
- Default: 800 words per chunk (roughly 800 tokens for English)
|
||||
- 15% overlap between chunks for context preservation
|
||||
- Minimum chunk size: 50 words (tiny trailing chunks merge with previous)
|
||||
@@ -95,7 +95,8 @@ async fn test_heartbeat_end_to_end() {
|
||||
println!("[6/6] Running check_heartbeat()...\n");
|
||||
|
||||
let hb_config = ironclaw::agent::HeartbeatConfig::default();
|
||||
let runner = HeartbeatRunner::new(hb_config, workspace, llm);
|
||||
let hygiene_config = ironclaw::workspace::hygiene::HygieneConfig::default();
|
||||
let runner = HeartbeatRunner::new(hb_config, hygiene_config, workspace, llm);
|
||||
|
||||
let result = runner.check_heartbeat().await;
|
||||
|
||||
|
||||
@@ -20,3 +20,5 @@ lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -19,3 +19,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -24,3 +24,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
Reference in New Issue
Block a user