mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-26 15:40:18 +00:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c75a5e2d4b | ||
|
|
cde50ff470 | ||
|
|
ca8d5c6b5e | ||
|
|
9fed8453c7 | ||
|
|
eaef335db6 | ||
|
|
225af29db2 | ||
|
|
a53b2c10b5 | ||
|
|
408ae8a29a | ||
|
|
d9ff86d7e0 | ||
|
|
e843c18141 | ||
|
|
54e9206f0b | ||
|
|
5df0d13b59 |
@@ -151,6 +151,12 @@ src/
|
|||||||
│ ├── rate_limiter.rs # Per-tool rate limiting
|
│ ├── rate_limiter.rs # Per-tool rate limiting
|
||||||
│ └── storage.rs # Linear memory persistence
|
│ └── storage.rs # Linear memory persistence
|
||||||
│
|
│
|
||||||
|
├── db/ # Database abstraction layer
|
||||||
|
│ ├── mod.rs # Database trait (~60 async methods)
|
||||||
|
│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository)
|
||||||
|
│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite)
|
||||||
|
│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent)
|
||||||
|
│
|
||||||
├── workspace/ # Persistent memory system (OpenClaw-inspired)
|
├── workspace/ # Persistent memory system (OpenClaw-inspired)
|
||||||
│ ├── mod.rs # Workspace struct, memory operations
|
│ ├── mod.rs # Workspace struct, memory operations
|
||||||
│ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry
|
│ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry
|
||||||
@@ -192,8 +198,9 @@ When designing new features or systems, always prefer generic/extensible archite
|
|||||||
|
|
||||||
### Error Handling
|
### Error Handling
|
||||||
- Use `thiserror` for error types in `error.rs`
|
- Use `thiserror` for error types in `error.rs`
|
||||||
- Never use `.unwrap()` in production code (tests are fine)
|
- Never use `.unwrap()` or `.expect()` in production code (tests are fine)
|
||||||
- Map errors with context: `.map_err(|e| SomeError::Variant { reason: e.to_string() })?`
|
- Map errors with context: `.map_err(|e| SomeError::Variant { reason: e.to_string() })?`
|
||||||
|
- Before committing, grep for `.unwrap()` and `.expect(` in changed files to catch violations mechanically
|
||||||
|
|
||||||
### Async
|
### Async
|
||||||
- All I/O is async with tokio
|
- All I/O is async with tokio
|
||||||
@@ -201,6 +208,7 @@ When designing new features or systems, always prefer generic/extensible archite
|
|||||||
- Use `RwLock` for concurrent read/write access
|
- Use `RwLock` for concurrent read/write access
|
||||||
|
|
||||||
### Traits for Extensibility
|
### Traits for Extensibility
|
||||||
|
- `Database` - Add new database backends (must implement all ~60 methods)
|
||||||
- `Channel` - Add new input sources
|
- `Channel` - Add new input sources
|
||||||
- `Tool` - Add new capabilities
|
- `Tool` - Add new capabilities
|
||||||
- `LlmProvider` - Add new LLM backends
|
- `LlmProvider` - Add new LLM backends
|
||||||
@@ -248,7 +256,12 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted
|
|||||||
|
|
||||||
Environment variables (see `.env.example`):
|
Environment variables (see `.env.example`):
|
||||||
```bash
|
```bash
|
||||||
|
# Database backend (default: postgres)
|
||||||
|
DATABASE_BACKEND=postgres # or "libsql" / "turso"
|
||||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||||
|
LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
|
||||||
|
# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional)
|
||||||
|
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
|
||||||
|
|
||||||
# NEAR AI (required)
|
# NEAR AI (required)
|
||||||
NEARAI_SESSION_TOKEN=sess_...
|
NEARAI_SESSION_TOKEN=sess_...
|
||||||
@@ -308,7 +321,51 @@ Session tokens have the format `sess_xxx` (37 characters). They are authenticate
|
|||||||
|
|
||||||
## Database
|
## Database
|
||||||
|
|
||||||
Single migration in `migrations/V1__initial.sql`. Tables:
|
IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable.
|
||||||
|
|
||||||
|
**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL).
|
||||||
|
|
||||||
|
### Backends
|
||||||
|
|
||||||
|
| Backend | Feature Flag | Default | Use Case |
|
||||||
|
|---------|-------------|---------|----------|
|
||||||
|
| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments |
|
||||||
|
| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud |
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build with PostgreSQL only (default)
|
||||||
|
cargo build
|
||||||
|
|
||||||
|
# Build with libSQL only
|
||||||
|
cargo build --no-default-features --features libsql
|
||||||
|
|
||||||
|
# Build with both backends available
|
||||||
|
cargo build --features "postgres,libsql"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Database Trait
|
||||||
|
|
||||||
|
The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence:
|
||||||
|
- Conversations, messages, metadata
|
||||||
|
- Jobs, actions, LLM calls, estimation snapshots
|
||||||
|
- Sandbox jobs, job events
|
||||||
|
- Routines, routine runs
|
||||||
|
- Tool failures, settings
|
||||||
|
- Workspace: documents, chunks, hybrid search
|
||||||
|
|
||||||
|
Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL.
|
||||||
|
|
||||||
|
### Schema
|
||||||
|
|
||||||
|
**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`.
|
||||||
|
|
||||||
|
**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types:
|
||||||
|
- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT`
|
||||||
|
- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx`
|
||||||
|
- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers
|
||||||
|
- PL/pgSQL functions -> SQLite triggers
|
||||||
|
|
||||||
|
**Tables (both backends):**
|
||||||
|
|
||||||
**Core:**
|
**Core:**
|
||||||
- `conversations` - Multi-channel conversation tracking
|
- `conversations` - Multi-channel conversation tracking
|
||||||
@@ -320,12 +377,41 @@ Single migration in `migrations/V1__initial.sql`. Tables:
|
|||||||
|
|
||||||
**Workspace/Memory:**
|
**Workspace/Memory:**
|
||||||
- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md")
|
- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md")
|
||||||
- `memory_chunks` - Chunked content with FTS (tsvector) and vector (pgvector) indexes
|
- `memory_chunks` - Chunked content with FTS and vector indexes
|
||||||
- `heartbeat_state` - Periodic execution tracking
|
- `heartbeat_state` - Periodic execution tracking
|
||||||
|
|
||||||
Requires pgvector extension: `CREATE EXTENSION IF NOT EXISTS vector;`
|
**Other:**
|
||||||
|
- `routines`, `routine_runs` - Scheduled/reactive execution
|
||||||
|
- `settings` - Per-user key-value settings
|
||||||
|
- `tool_failures` - Self-repair tracking
|
||||||
|
- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure
|
||||||
|
|
||||||
Run migrations: `refinery migrate -c refinery.toml`
|
### Configuration
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Backend selection (default: postgres)
|
||||||
|
DATABASE_BACKEND=libsql
|
||||||
|
|
||||||
|
# PostgreSQL
|
||||||
|
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||||
|
|
||||||
|
# libSQL (embedded)
|
||||||
|
LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path
|
||||||
|
|
||||||
|
# libSQL (Turso cloud sync)
|
||||||
|
LIBSQL_URL=libsql://your-db.turso.io
|
||||||
|
LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set
|
||||||
|
```
|
||||||
|
|
||||||
|
### Current Limitations (libSQL backend)
|
||||||
|
|
||||||
|
- **Workspace/memory system** not yet wired through Database trait (requires Store migration)
|
||||||
|
- **Secrets store** not yet available (still requires PostgresSecretsStore)
|
||||||
|
- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented)
|
||||||
|
- **Settings reload from DB** skipped (Config::from_db requires Store)
|
||||||
|
- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet)
|
||||||
|
- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage.
|
||||||
|
- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields.
|
||||||
|
|
||||||
## Safety Layer
|
## Safety Layer
|
||||||
|
|
||||||
@@ -387,6 +473,7 @@ Key test patterns:
|
|||||||
- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers
|
- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers
|
||||||
- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails
|
- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails
|
||||||
- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI
|
- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI
|
||||||
|
- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode
|
||||||
|
|
||||||
## Adding a New Tool
|
## Adding a New Tool
|
||||||
|
|
||||||
@@ -551,6 +638,37 @@ RUST_LOG=ironclaw=debug,tower_http=debug cargo run
|
|||||||
- Keep functions focused, extract helpers when logic is reused
|
- Keep functions focused, extract helpers when logic is reused
|
||||||
- Comments for non-obvious logic only
|
- Comments for non-obvious logic only
|
||||||
|
|
||||||
|
## Review & Fix Discipline
|
||||||
|
|
||||||
|
Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback.
|
||||||
|
|
||||||
|
### Fix the pattern, not just the instance
|
||||||
|
When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix.
|
||||||
|
|
||||||
|
### Propagate architectural fixes to satellite types
|
||||||
|
If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase.
|
||||||
|
|
||||||
|
### Schema translation is more than DDL
|
||||||
|
When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for:
|
||||||
|
- **Indexes** -- diff `CREATE INDEX` statements between the two schemas
|
||||||
|
- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`)
|
||||||
|
- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`)
|
||||||
|
|
||||||
|
### Feature flag testing
|
||||||
|
When adding feature-gated code, test compilation with each feature in isolation:
|
||||||
|
```bash
|
||||||
|
cargo check # default features
|
||||||
|
cargo check --no-default-features --features libsql # libsql only
|
||||||
|
cargo check --all-features # all features
|
||||||
|
```
|
||||||
|
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||||
|
|
||||||
|
### Mechanical verification before committing
|
||||||
|
Run these checks on changed files before committing:
|
||||||
|
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||||
|
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||||
|
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||||
|
|
||||||
## Workspace & Memory System
|
## Workspace & Memory System
|
||||||
|
|
||||||
Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure.
|
Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure.
|
||||||
@@ -625,7 +743,7 @@ Four tools for LLM use:
|
|||||||
|
|
||||||
### Hybrid Search (RRF)
|
### Hybrid Search (RRF)
|
||||||
|
|
||||||
Combines full-text search (PostgreSQL `ts_rank_cd`) and vector similarity (pgvector cosine) using Reciprocal Rank Fusion:
|
Combines full-text search and vector similarity using Reciprocal Rank Fusion:
|
||||||
|
|
||||||
```
|
```
|
||||||
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||||
@@ -633,6 +751,10 @@ score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
|||||||
|
|
||||||
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
||||||
|
|
||||||
|
**Backend differences:**
|
||||||
|
- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF
|
||||||
|
- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired)
|
||||||
|
|
||||||
### Heartbeat System
|
### Heartbeat System
|
||||||
|
|
||||||
Proactive periodic execution (default: 30 minutes):
|
Proactive periodic execution (default: 30 minutes):
|
||||||
|
|||||||
Generated
+804
-82
File diff suppressed because it is too large
Load Diff
+31
-11
@@ -2,7 +2,7 @@
|
|||||||
name = "ironclaw"
|
name = "ironclaw"
|
||||||
version = "0.1.3"
|
version = "0.1.3"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
rust-version = "1.85"
|
rust-version = "1.92"
|
||||||
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
||||||
authors = ["NEAR AI <[email protected]>"]
|
authors = ["NEAR AI <[email protected]>"]
|
||||||
license = "MIT OR Apache-2.0"
|
license = "MIT OR Apache-2.0"
|
||||||
@@ -22,17 +22,20 @@ tokio-stream = { version = "0.1", features = ["sync"] }
|
|||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
|
|
||||||
# HTTP client
|
# HTTP client
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
|
||||||
|
|
||||||
# Serialization
|
# Serialization
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|
||||||
# Database
|
# Database - PostgreSQL (default, feature-gated)
|
||||||
deadpool-postgres = "0.14"
|
deadpool-postgres = { version = "0.14", optional = true }
|
||||||
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"] }
|
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true }
|
||||||
postgres-types = { version = "0.2", features = ["with-serde_json-1"] }
|
postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true }
|
||||||
refinery = { version = "0.8", features = ["tokio-postgres"] }
|
refinery = { version = "0.8", features = ["tokio-postgres"], optional = true }
|
||||||
|
|
||||||
|
# Database - libSQL/Turso (optional embedded database)
|
||||||
|
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] }
|
||||||
|
|
||||||
# Error handling
|
# Error handling
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
@@ -48,7 +51,7 @@ dotenvy = "0.15"
|
|||||||
# Core types
|
# Core types
|
||||||
uuid = { version = "1", features = ["v4", "serde"] }
|
uuid = { version = "1", features = ["v4", "serde"] }
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "db-tokio-postgres", "maths"] }
|
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] }
|
||||||
rust_decimal_macros = "1"
|
rust_decimal_macros = "1"
|
||||||
|
|
||||||
# Async traits
|
# Async traits
|
||||||
@@ -81,7 +84,8 @@ fs4 = "0.6"
|
|||||||
# Secrecy for sensitive values
|
# Secrecy for sensitive values
|
||||||
secrecy = { version = "0.10", features = ["serde"] }
|
secrecy = { version = "0.10", features = ["serde"] }
|
||||||
|
|
||||||
# URL encoding for OAuth flow
|
# URL parsing and encoding
|
||||||
|
url = "2"
|
||||||
urlencoding = "2"
|
urlencoding = "2"
|
||||||
|
|
||||||
# Open URLs in browser
|
# Open URLs in browser
|
||||||
@@ -89,7 +93,7 @@ open = "5"
|
|||||||
|
|
||||||
# Vector embeddings for semantic search
|
# Vector embeddings for semantic search
|
||||||
# The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres)
|
# The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres)
|
||||||
pgvector = { version = "0.4", features = ["postgres"] }
|
pgvector = { version = "0.4", features = ["postgres"], optional = true }
|
||||||
|
|
||||||
# WASM sandbox for untrusted tool execution
|
# WASM sandbox for untrusted tool execution
|
||||||
wasmtime = { version = "28", features = ["component-model"] }
|
wasmtime = { version = "28", features = ["component-model"] }
|
||||||
@@ -118,6 +122,9 @@ bytes = "1"
|
|||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
mime_guess = "2.0.5"
|
mime_guess = "2.0.5"
|
||||||
|
|
||||||
|
# Headless browser automation via Chrome DevTools Protocol
|
||||||
|
chromiumoxide = { version = "0.8", default-features = false, features = ["tokio-runtime"] }
|
||||||
|
|
||||||
# macOS keychain
|
# macOS keychain
|
||||||
[target.'cfg(target_os = "macos")'.dependencies]
|
[target.'cfg(target_os = "macos")'.dependencies]
|
||||||
security-framework = "3"
|
security-framework = "3"
|
||||||
@@ -135,9 +142,22 @@ pretty_assertions = "1"
|
|||||||
tempfile = "3"
|
tempfile = "3"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = ["postgres"]
|
||||||
|
postgres = [
|
||||||
|
"dep:deadpool-postgres",
|
||||||
|
"dep:tokio-postgres",
|
||||||
|
"dep:postgres-types",
|
||||||
|
"dep:refinery",
|
||||||
|
"dep:pgvector",
|
||||||
|
"rust_decimal/db-tokio-postgres",
|
||||||
|
]
|
||||||
|
libsql = ["dep:libsql"]
|
||||||
integration = []
|
integration = []
|
||||||
|
|
||||||
|
[[example]]
|
||||||
|
name = "test_heartbeat"
|
||||||
|
required-features = ["postgres"]
|
||||||
|
|
||||||
# The profile that 'cargo dist' will build with
|
# The profile that 'cargo dist' will build with
|
||||||
[profile.dist]
|
[profile.dist]
|
||||||
inherits = "release"
|
inherits = "release"
|
||||||
|
|||||||
+46
@@ -0,0 +1,46 @@
|
|||||||
|
# Multi-stage Dockerfile for the IronClaw agent (cloud deployment).
|
||||||
|
#
|
||||||
|
# Build:
|
||||||
|
# docker build --platform linux/amd64 -t ironclaw:latest .
|
||||||
|
#
|
||||||
|
# Run:
|
||||||
|
# docker run --env-file .env -p 3000:3000 ironclaw:latest
|
||||||
|
|
||||||
|
# Stage 1: Build
|
||||||
|
FROM rust:1.92-slim-bookworm AS builder
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
pkg-config libssl-dev cmake gcc g++ \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy manifests first for layer caching
|
||||||
|
COPY Cargo.toml Cargo.lock ./
|
||||||
|
|
||||||
|
# Copy source and build artifacts
|
||||||
|
COPY src/ src/
|
||||||
|
COPY migrations/ migrations/
|
||||||
|
COPY wit/ wit/
|
||||||
|
|
||||||
|
RUN cargo build --release --bin ironclaw
|
||||||
|
|
||||||
|
# Stage 2: Runtime
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates libssl3 \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=builder /app/target/release/ironclaw /usr/local/bin/ironclaw
|
||||||
|
COPY --from=builder /app/migrations /app/migrations
|
||||||
|
|
||||||
|
# Non-root user
|
||||||
|
RUN useradd -m -u 1000 -s /bin/bash ironclaw
|
||||||
|
USER ironclaw
|
||||||
|
|
||||||
|
EXPOSE 3000
|
||||||
|
|
||||||
|
ENV RUST_LOG=ironclaw=info
|
||||||
|
|
||||||
|
ENTRYPOINT ["ironclaw"]
|
||||||
+2
-2
@@ -9,7 +9,7 @@
|
|||||||
# The image includes common development tools so workers can build software,
|
# The image includes common development tools so workers can build software,
|
||||||
# run tests, and execute shell commands.
|
# run tests, and execute shell commands.
|
||||||
|
|
||||||
FROM rust:1.85-bookworm AS builder
|
FROM rust:1.92-bookworm AS builder
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
COPY . .
|
COPY . .
|
||||||
@@ -40,7 +40,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
ENV RUSTUP_HOME=/usr/local/rustup \
|
ENV RUSTUP_HOME=/usr/local/rustup \
|
||||||
CARGO_HOME=/usr/local/cargo \
|
CARGO_HOME=/usr/local/cargo \
|
||||||
PATH=/usr/local/cargo/bin:$PATH
|
PATH=/usr/local/cargo/bin:$PATH
|
||||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.85.0 \
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.92.0 \
|
||||||
&& chmod -R a+r /usr/local/rustup /usr/local/cargo
|
&& chmod -R a+r /usr/local/rustup /usr/local/cargo
|
||||||
|
|
||||||
# Install Claude Code CLI (for claude-bridge mode)
|
# Install Claude Code CLI (for claude-bridge mode)
|
||||||
|
|||||||
+3
-3
@@ -133,7 +133,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
|||||||
|---------|----------|----------|-------|
|
|---------|----------|----------|-------|
|
||||||
| Pi agent runtime | ✅ | ➖ | IronClaw uses custom runtime |
|
| Pi agent runtime | ✅ | ➖ | IronClaw uses custom runtime |
|
||||||
| RPC-based execution | ✅ | ✅ | Orchestrator/worker pattern |
|
| RPC-based execution | ✅ | ✅ | Orchestrator/worker pattern |
|
||||||
| Multi-provider failover | ✅ | ❌ | Provider fallback chains |
|
| Multi-provider failover | ✅ | ✅ | `FailoverProvider` tries providers sequentially on retryable errors |
|
||||||
| Per-sender sessions | ✅ | ✅ | |
|
| Per-sender sessions | ✅ | ✅ | |
|
||||||
| Global sessions | ✅ | ❌ | Optional shared context |
|
| Global sessions | ✅ | ❌ | Optional shared context |
|
||||||
| Session pruning | ✅ | ❌ | Auto cleanup old sessions |
|
| Session pruning | ✅ | ❌ | Auto cleanup old sessions |
|
||||||
@@ -173,7 +173,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
|||||||
| Feature | OpenClaw | IronClaw | Notes |
|
| Feature | OpenClaw | IronClaw | Notes |
|
||||||
|---------|----------|----------|-------|
|
|---------|----------|----------|-------|
|
||||||
| Auto-discovery | ✅ | ❌ | |
|
| Auto-discovery | ✅ | ❌ | |
|
||||||
| Failover chains | ✅ | ❌ | Provider fallback |
|
| Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` |
|
||||||
| Cooldown management | ✅ | ❌ | Skip failed providers |
|
| Cooldown management | ✅ | ❌ | Skip failed providers |
|
||||||
| Per-session model override | ✅ | ✅ | Model selector in TUI |
|
| Per-session model override | ✅ | ✅ | Model selector in TUI |
|
||||||
| Model selection UI | ✅ | ✅ | TUI keyboard shortcut |
|
| Model selection UI | ✅ | ✅ | TUI keyboard shortcut |
|
||||||
@@ -419,7 +419,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
|||||||
- ❌ Slack channel (real implementation)
|
- ❌ Slack channel (real implementation)
|
||||||
- ✅ Telegram channel (WASM, DM pairing, caption, /start)
|
- ✅ Telegram channel (WASM, DM pairing, caption, /start)
|
||||||
- ❌ WhatsApp channel
|
- ❌ WhatsApp channel
|
||||||
- ❌ Multi-provider failover
|
- ✅ Multi-provider failover (`FailoverProvider` with retryable error classification)
|
||||||
- ❌ Hooks system (beforeInbound, beforeToolCall, etc.)
|
- ❌ Hooks system (beforeInbound, beforeToolCall, etc.)
|
||||||
|
|
||||||
### P2 - Medium Priority
|
### P2 - Medium Priority
|
||||||
|
|||||||
@@ -181,42 +181,42 @@ External content passes through multiple security layers:
|
|||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
┌────────────────────────────────────────────────────────────────────┐
|
┌────────────────────────────────────────────────────────────────┐
|
||||||
│ Channels │
|
│ Channels │
|
||||||
│ ┌──────┐ ┌──────┐ ┌─────────────┐ ┌─────────────┐ │
|
│ ┌──────┐ ┌──────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||||
│ │ REPL │ │ HTTP │ │WASM Channels│ │ Web Gateway │ │
|
│ │ REPL │ │ HTTP │ │WASM Channels│ │ Web Gateway │ │
|
||||||
│ └──┬───┘ └──┬───┘ └──────┬──────┘ │ (SSE + WS) │ │
|
│ └──┬───┘ └──┬───┘ └──────┬──────┘ │ (SSE + WS) │ │
|
||||||
│ │ │ │ └──────┬──────┘ │
|
│ │ │ │ └──────┬──────┘ │
|
||||||
│ └─────────┴──────────────┴────────────────┘ │
|
│ └─────────┴──────────────┴────────────────┘ │
|
||||||
│ │ │
|
│ │ │
|
||||||
│ ┌─────────▼─────────┐ │
|
│ ┌─────────▼─────────┐ │
|
||||||
│ │ Agent Loop │ Intent routing │
|
│ │ Agent Loop │ Intent routing │
|
||||||
│ └────┬─────────┬────┘ │
|
│ └────┬──────────┬───┘ │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ ┌──────────▼───┐ ┌──▼──────────────┐ │
|
│ ┌──────────▼────┐ ┌──▼───────────────┐ │
|
||||||
│ │ Scheduler │ │ Routines Engine │ │
|
│ │ Scheduler │ │ Routines Engine │ │
|
||||||
│ │(parallel jobs)│ │(cron, event, wh) │ │
|
│ │(parallel jobs)│ │(cron, event, wh) │ │
|
||||||
│ └──────┬───────┘ └────────┬─────────┘ │
|
│ └──────┬────────┘ └────────┬─────────┘ │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ ┌─────────────┼───────────────────┘ │
|
│ ┌─────────────┼────────────────────┘ │
|
||||||
│ │ │ │
|
│ │ │ │
|
||||||
│ ┌───▼────┐ ┌────▼────────────────┐ │
|
│ ┌───▼─────┐ ┌────▼────────────────┐ │
|
||||||
│ │ Local │ │ Orchestrator │ │
|
│ │ Local │ │ Orchestrator │ │
|
||||||
│ │Workers │ │ ┌───────────────┐ │ │
|
│ │Workers │ │ ┌───────────────┐ │ │
|
||||||
│ │(in-proc)│ │ │ Docker Sandbox│ │ │
|
│ │(in-proc)│ │ │ Docker Sandbox│ │ │
|
||||||
│ └───┬────┘ │ │ Containers │ │ │
|
│ └───┬─────┘ │ │ Containers │ │ │
|
||||||
│ │ │ │ ┌───────────┐ │ │ │
|
│ │ │ │ ┌───────────┐ │ │ │
|
||||||
│ │ │ │ │Worker / CC│ │ │ │
|
│ │ │ │ │Worker / CC│ │ │ │
|
||||||
│ │ │ │ └───────────┘ │ │ │
|
│ │ │ │ └───────────┘ │ │ │
|
||||||
│ │ │ └───────────────┘ │ │
|
│ │ │ └───────────────┘ │ │
|
||||||
│ │ └─────────┬───────────┘ │
|
│ │ └─────────┬───────────┘ │
|
||||||
│ └──────────────────┤ │
|
│ └──────────────────┤ │
|
||||||
│ │ │
|
│ │ │
|
||||||
│ ┌───────────▼──────────┐ │
|
│ ┌───────────▼──────────┐ │
|
||||||
│ │ Tool Registry │ │
|
│ │ Tool Registry │ │
|
||||||
│ │ Built-in, MCP, WASM │ │
|
│ │ Built-in, MCP, WASM │ │
|
||||||
│ └──────────────────────┘ │
|
│ └──────────────────────┘ │
|
||||||
└────────────────────────────────────────────────────────────────────┘
|
└────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
### Core Components
|
### Core Components
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Cloud SQL Auth Proxy
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
DynamicUser=yes
|
||||||
|
ExecStart=/usr/local/bin/cloud-sql-proxy ironclaw-prod:us-central1:ironclaw-db --port=5432
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# WARNING: Replace all CHANGE_ME values before deploying.
|
||||||
|
# Do not use placeholder passwords in production.
|
||||||
|
DATABASE_URL=postgres://ironclaw:CHANGE_ME@localhost:5432/ironclaw
|
||||||
|
|
||||||
|
# NEAR AI
|
||||||
|
NEARAI_SESSION_TOKEN=CHANGE_ME
|
||||||
|
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||||
|
NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||||
|
NEARAI_AUTH_URL=https://private.near.ai
|
||||||
|
NEARAI_API_MODE=chat_completions
|
||||||
|
|
||||||
|
# Agent
|
||||||
|
AGENT_NAME=ironclaw
|
||||||
|
CLI_ENABLED=false
|
||||||
|
|
||||||
|
# Web Gateway
|
||||||
|
GATEWAY_ENABLED=true
|
||||||
|
# 0.0.0.0 binds to all interfaces (required for Docker --network=host).
|
||||||
|
# Use 127.0.0.1 if running outside Docker or for local-only access.
|
||||||
|
GATEWAY_HOST=0.0.0.0
|
||||||
|
GATEWAY_PORT=3000
|
||||||
|
GATEWAY_AUTH_TOKEN=CHANGE_ME
|
||||||
|
|
||||||
|
# Disabled for initial deploy
|
||||||
|
SANDBOX_ENABLED=false
|
||||||
|
HEARTBEAT_ENABLED=false
|
||||||
|
EMBEDDING_ENABLED=false
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=IronClaw AI Assistant
|
||||||
|
After=cloud-sql-proxy.service docker.service
|
||||||
|
Requires=cloud-sql-proxy.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
ExecStartPre=/usr/bin/docker pull us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest
|
||||||
|
ExecStart=/usr/bin/docker run --rm \
|
||||||
|
--name ironclaw \
|
||||||
|
--env-file /opt/ironclaw/.env \
|
||||||
|
--network=host \
|
||||||
|
us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest \
|
||||||
|
--no-onboard
|
||||||
|
ExecStop=/usr/bin/docker stop ironclaw
|
||||||
|
Restart=always
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
Executable
+68
@@ -0,0 +1,68 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# VM bootstrap script for IronClaw on GCP Compute Engine.
|
||||||
|
#
|
||||||
|
# Run on a fresh Debian 12 VM after SSH:
|
||||||
|
# sudo bash setup.sh
|
||||||
|
#
|
||||||
|
# Prerequisites:
|
||||||
|
# - VM has the ironclaw-vm service account attached
|
||||||
|
# - Cloud SQL Auth Proxy accessible via IAM
|
||||||
|
# - Artifact Registry image pushed
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Must run as root
|
||||||
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
|
echo "ERROR: This script must be run as root (sudo bash setup.sh)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Installing Docker"
|
||||||
|
apt-get update
|
||||||
|
apt-get install -y docker.io
|
||||||
|
systemctl enable docker
|
||||||
|
systemctl start docker
|
||||||
|
|
||||||
|
echo "==> Installing Cloud SQL Auth Proxy"
|
||||||
|
curl -fsSL -o /usr/local/bin/cloud-sql-proxy \
|
||||||
|
https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.14.3/cloud-sql-proxy.linux.amd64
|
||||||
|
chmod +x /usr/local/bin/cloud-sql-proxy
|
||||||
|
|
||||||
|
echo "==> Installing systemd services"
|
||||||
|
cp /tmp/deploy/cloud-sql-proxy.service /etc/systemd/system/
|
||||||
|
cp /tmp/deploy/ironclaw.service /etc/systemd/system/
|
||||||
|
systemctl daemon-reload
|
||||||
|
|
||||||
|
echo "==> Starting Cloud SQL Auth Proxy"
|
||||||
|
systemctl enable cloud-sql-proxy
|
||||||
|
systemctl start cloud-sql-proxy
|
||||||
|
|
||||||
|
echo "==> Configuring Docker registry auth"
|
||||||
|
# The VM service account provides Artifact Registry access
|
||||||
|
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
||||||
|
|
||||||
|
echo "==> Creating config directory"
|
||||||
|
# Owned by root, readable only by root. Docker reads --env-file as root
|
||||||
|
# before dropping to uid 1000 (ironclaw) inside the container.
|
||||||
|
mkdir -p /opt/ironclaw
|
||||||
|
chmod 700 /opt/ironclaw
|
||||||
|
|
||||||
|
if [ ! -f /opt/ironclaw/.env ]; then
|
||||||
|
echo "WARNING: /opt/ironclaw/.env does not exist."
|
||||||
|
echo "Create it with your configuration before starting IronClaw."
|
||||||
|
echo "See deploy/env.example for the required variables."
|
||||||
|
echo ""
|
||||||
|
echo "Then run: systemctl enable ironclaw && systemctl start ironclaw"
|
||||||
|
else
|
||||||
|
chmod 600 /opt/ironclaw/.env
|
||||||
|
echo "==> Starting IronClaw"
|
||||||
|
systemctl enable ironclaw
|
||||||
|
systemctl start ironclaw
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "==> Setup complete"
|
||||||
|
echo ""
|
||||||
|
echo "Verify with:"
|
||||||
|
echo " systemctl status cloud-sql-proxy"
|
||||||
|
echo " systemctl status ironclaw"
|
||||||
|
echo " docker logs ironclaw"
|
||||||
@@ -81,7 +81,6 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
let session = create_session_manager(SessionConfig {
|
let session = create_session_manager(SessionConfig {
|
||||||
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
||||||
session_path: config.llm.nearai.session_path.clone(),
|
session_path: config.llm.nearai.session_path.clone(),
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
let llm = create_llm_provider(&config.llm, session)?;
|
let llm = create_llm_provider(&config.llm, session)?;
|
||||||
|
|||||||
+222
-151
@@ -19,16 +19,16 @@ use crate::channels::{ChannelManager, IncomingMessage, OutgoingResponse, StatusU
|
|||||||
use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig};
|
use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig};
|
||||||
use crate::context::ContextManager;
|
use crate::context::ContextManager;
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
|
use crate::db::Database;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::extensions::ExtensionManager;
|
use crate::extensions::ExtensionManager;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult};
|
use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult};
|
||||||
use crate::safety::SafetyLayer;
|
use crate::safety::SafetyLayer;
|
||||||
use crate::tools::ToolRegistry;
|
use crate::tools::ToolRegistry;
|
||||||
use crate::workspace::Workspace;
|
use crate::workspace::Workspace;
|
||||||
|
|
||||||
/// Collapse a tool output string into a single-line preview for display.
|
/// Collapse a tool output string into a single-line preview for display.
|
||||||
fn truncate_for_preview(output: &str, max_chars: usize) -> String {
|
pub(crate) fn truncate_for_preview(output: &str, max_chars: usize) -> String {
|
||||||
let collapsed: String = output
|
let collapsed: String = output
|
||||||
.chars()
|
.chars()
|
||||||
.take(max_chars + 50)
|
.take(max_chars + 50)
|
||||||
@@ -37,8 +37,14 @@ fn truncate_for_preview(output: &str, max_chars: usize) -> String {
|
|||||||
.split_whitespace()
|
.split_whitespace()
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(" ");
|
.join(" ");
|
||||||
if collapsed.len() > max_chars {
|
// char_indices gives us byte offsets at char boundaries, so the slice is always valid UTF-8.
|
||||||
format!("{}...", &collapsed[..max_chars])
|
if collapsed.chars().count() > max_chars {
|
||||||
|
let byte_offset = collapsed
|
||||||
|
.char_indices()
|
||||||
|
.nth(max_chars)
|
||||||
|
.map(|(i, _)| i)
|
||||||
|
.unwrap_or(collapsed.len());
|
||||||
|
format!("{}...", &collapsed[..byte_offset])
|
||||||
} else {
|
} else {
|
||||||
collapsed
|
collapsed
|
||||||
}
|
}
|
||||||
@@ -59,7 +65,7 @@ enum AgenticLoopResult {
|
|||||||
///
|
///
|
||||||
/// Bundles the shared components to reduce argument count.
|
/// Bundles the shared components to reduce argument count.
|
||||||
pub struct AgentDeps {
|
pub struct AgentDeps {
|
||||||
pub store: Option<Arc<Store>>,
|
pub store: Option<Arc<dyn Database>>,
|
||||||
pub llm: Arc<dyn LlmProvider>,
|
pub llm: Arc<dyn LlmProvider>,
|
||||||
pub safety: Arc<SafetyLayer>,
|
pub safety: Arc<SafetyLayer>,
|
||||||
pub tools: Arc<ToolRegistry>,
|
pub tools: Arc<ToolRegistry>,
|
||||||
@@ -124,7 +130,7 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Convenience accessors
|
// Convenience accessors
|
||||||
fn store(&self) -> Option<&Arc<Store>> {
|
fn store(&self) -> Option<&Arc<dyn Database>> {
|
||||||
self.deps.store.as_ref()
|
self.deps.store.as_ref()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -654,19 +660,17 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Restore response chain from conversation metadata
|
// Restore response chain from conversation metadata
|
||||||
if let Some(store) = self.store() {
|
if let Some(store) = self.store()
|
||||||
if let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await {
|
&& let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await
|
||||||
if let Some(rid) = metadata
|
&& let Some(rid) = metadata
|
||||||
.get("last_response_id")
|
.get("last_response_id")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.map(String::from)
|
.map(String::from)
|
||||||
{
|
{
|
||||||
thread.last_response_id = Some(rid.clone());
|
thread.last_response_id = Some(rid.clone());
|
||||||
self.llm()
|
self.llm()
|
||||||
.seed_response_chain(&thread_uuid.to_string(), rid);
|
.seed_response_chain(&thread_uuid.to_string(), rid);
|
||||||
tracing::debug!("Restored response chain for thread {}", thread_uuid);
|
tracing::debug!("Restored response chain for thread {}", thread_uuid);
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert into session and register with session manager
|
// Insert into session and register with session manager
|
||||||
@@ -954,13 +958,12 @@ impl Agent {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref resp) = response {
|
if let Some(ref resp) = response
|
||||||
if let Err(e) = store
|
&& let Err(e) = store
|
||||||
.add_conversation_message(thread_id, "assistant", resp)
|
.add_conversation_message(thread_id, "assistant", resp)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!("Failed to persist assistant message: {}", e);
|
tracing::warn!("Failed to persist assistant message: {}", e);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1058,14 +1061,14 @@ impl Agent {
|
|||||||
// Check if interrupted
|
// Check if interrupted
|
||||||
{
|
{
|
||||||
let sess = session.lock().await;
|
let sess = session.lock().await;
|
||||||
if let Some(thread) = sess.threads.get(&thread_id) {
|
if let Some(thread) = sess.threads.get(&thread_id)
|
||||||
if thread.state == ThreadState::Interrupted {
|
&& thread.state == ThreadState::Interrupted
|
||||||
return Err(crate::error::JobError::ContextError {
|
{
|
||||||
id: thread_id,
|
return Err(crate::error::JobError::ContextError {
|
||||||
reason: "Interrupted".to_string(),
|
id: thread_id,
|
||||||
}
|
reason: "Interrupted".to_string(),
|
||||||
.into());
|
|
||||||
}
|
}
|
||||||
|
.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1140,11 +1143,11 @@ impl Agent {
|
|||||||
// Record tool calls in the thread
|
// Record tool calls in the thread
|
||||||
{
|
{
|
||||||
let mut sess = session.lock().await;
|
let mut sess = session.lock().await;
|
||||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
if let Some(thread) = sess.threads.get_mut(&thread_id)
|
||||||
if let Some(turn) = thread.last_turn_mut() {
|
&& let Some(turn) = thread.last_turn_mut()
|
||||||
for tc in &tool_calls {
|
{
|
||||||
turn.record_tool_call(&tc.name, tc.arguments.clone());
|
for tc in &tool_calls {
|
||||||
}
|
turn.record_tool_call(&tc.name, tc.arguments.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1152,54 +1155,56 @@ impl Agent {
|
|||||||
// Execute each tool (with approval checking)
|
// Execute each tool (with approval checking)
|
||||||
for tc in tool_calls {
|
for tc in tool_calls {
|
||||||
// Check if tool requires approval
|
// Check if tool requires approval
|
||||||
if let Some(tool) = self.tools().get(&tc.name).await {
|
if let Some(tool) = self.tools().get(&tc.name).await
|
||||||
if tool.requires_approval() {
|
&& tool.requires_approval()
|
||||||
// Check if auto-approved for this session
|
{
|
||||||
let mut is_auto_approved = {
|
// Check if auto-approved for this session
|
||||||
let sess = session.lock().await;
|
let mut is_auto_approved = {
|
||||||
sess.is_tool_auto_approved(&tc.name)
|
let sess = session.lock().await;
|
||||||
|
sess.is_tool_auto_approved(&tc.name)
|
||||||
|
};
|
||||||
|
|
||||||
|
// For shell commands, override auto-approval for
|
||||||
|
// destructive patterns that should always require
|
||||||
|
// explicit per-invocation approval.
|
||||||
|
if is_auto_approved
|
||||||
|
&& tc.name == "shell"
|
||||||
|
&& let Some(cmd) = tc
|
||||||
|
.arguments
|
||||||
|
.get("command")
|
||||||
|
.and_then(|c| c.as_str().map(String::from))
|
||||||
|
.or_else(|| {
|
||||||
|
tc.arguments
|
||||||
|
.as_str()
|
||||||
|
.and_then(|s| {
|
||||||
|
serde_json::from_str::<serde_json::Value>(s).ok()
|
||||||
|
})
|
||||||
|
.and_then(|v| {
|
||||||
|
v.get("command")
|
||||||
|
.and_then(|c| c.as_str().map(String::from))
|
||||||
|
})
|
||||||
|
})
|
||||||
|
&& crate::tools::builtin::shell::requires_explicit_approval(&cmd)
|
||||||
|
{
|
||||||
|
tracing::info!(
|
||||||
|
"Shell command '{}' requires explicit approval despite auto-approve",
|
||||||
|
cmd.chars().take(80).collect::<String>()
|
||||||
|
);
|
||||||
|
is_auto_approved = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !is_auto_approved {
|
||||||
|
// Need approval - store pending request and return
|
||||||
|
let pending = PendingApproval {
|
||||||
|
request_id: Uuid::new_v4(),
|
||||||
|
tool_name: tc.name.clone(),
|
||||||
|
parameters: tc.arguments.clone(),
|
||||||
|
description: tool.description().to_string(),
|
||||||
|
tool_call_id: tc.id.clone(),
|
||||||
|
context_messages: context_messages.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// For shell commands, override auto-approval for
|
return Ok(AgenticLoopResult::NeedApproval { pending });
|
||||||
// destructive patterns that should always require
|
|
||||||
// explicit per-invocation approval.
|
|
||||||
if is_auto_approved && tc.name == "shell" {
|
|
||||||
if let Some(cmd) = tc
|
|
||||||
.arguments
|
|
||||||
.as_str()
|
|
||||||
.and_then(|s| {
|
|
||||||
serde_json::from_str::<serde_json::Value>(s).ok()
|
|
||||||
})
|
|
||||||
.and_then(|v| {
|
|
||||||
v.get("command")
|
|
||||||
.and_then(|c| c.as_str().map(String::from))
|
|
||||||
})
|
|
||||||
{
|
|
||||||
if crate::tools::builtin::shell::requires_explicit_approval(
|
|
||||||
&cmd,
|
|
||||||
) {
|
|
||||||
tracing::info!(
|
|
||||||
"Shell command '{}' requires explicit approval despite auto-approve",
|
|
||||||
cmd.chars().take(80).collect::<String>()
|
|
||||||
);
|
|
||||||
is_auto_approved = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !is_auto_approved {
|
|
||||||
// Need approval - store pending request and return
|
|
||||||
let pending = PendingApproval {
|
|
||||||
request_id: Uuid::new_v4(),
|
|
||||||
tool_name: tc.name.clone(),
|
|
||||||
parameters: tc.arguments.clone(),
|
|
||||||
description: tool.description().to_string(),
|
|
||||||
tool_call_id: tc.id.clone(),
|
|
||||||
context_messages: context_messages.clone(),
|
|
||||||
};
|
|
||||||
|
|
||||||
return Ok(AgenticLoopResult::NeedApproval { pending });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1230,34 +1235,34 @@ impl Agent {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
if let Ok(ref output) = tool_result {
|
if let Ok(ref output) = tool_result
|
||||||
if !output.is_empty() {
|
&& !output.is_empty()
|
||||||
let _ = self
|
{
|
||||||
.channels
|
let _ = self
|
||||||
.send_status(
|
.channels
|
||||||
&message.channel,
|
.send_status(
|
||||||
StatusUpdate::ToolResult {
|
&message.channel,
|
||||||
name: tc.name.clone(),
|
StatusUpdate::ToolResult {
|
||||||
preview: truncate_for_preview(output, 200),
|
name: tc.name.clone(),
|
||||||
},
|
preview: output.clone(),
|
||||||
&message.metadata,
|
},
|
||||||
)
|
&message.metadata,
|
||||||
.await;
|
)
|
||||||
}
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Record result in thread
|
// Record result in thread
|
||||||
{
|
{
|
||||||
let mut sess = session.lock().await;
|
let mut sess = session.lock().await;
|
||||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
if let Some(thread) = sess.threads.get_mut(&thread_id)
|
||||||
if let Some(turn) = thread.last_turn_mut() {
|
&& let Some(turn) = thread.last_turn_mut()
|
||||||
match &tool_result {
|
{
|
||||||
Ok(output) => {
|
match &tool_result {
|
||||||
turn.record_tool_result(serde_json::json!(output));
|
Ok(output) => {
|
||||||
}
|
turn.record_tool_result(serde_json::json!(output));
|
||||||
Err(e) => {
|
}
|
||||||
turn.record_tool_error(e.to_string());
|
Err(e) => {
|
||||||
}
|
turn.record_tool_error(e.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1640,17 +1645,17 @@ impl Agent {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Verify request ID if provided
|
// Verify request ID if provided
|
||||||
if let Some(req_id) = request_id {
|
if let Some(req_id) = request_id
|
||||||
if req_id != pending.request_id {
|
&& req_id != pending.request_id
|
||||||
// Put it back and return error
|
{
|
||||||
let mut sess = session.lock().await;
|
// Put it back and return error
|
||||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
let mut sess = session.lock().await;
|
||||||
thread.await_approval(pending);
|
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
||||||
}
|
thread.await_approval(pending);
|
||||||
return Ok(SubmissionResult::error(
|
|
||||||
"Request ID mismatch. Use the correct request ID.",
|
|
||||||
));
|
|
||||||
}
|
}
|
||||||
|
return Ok(SubmissionResult::error(
|
||||||
|
"Request ID mismatch. Use the correct request ID.",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if approved {
|
if approved {
|
||||||
@@ -1704,20 +1709,20 @@ impl Agent {
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
if let Ok(ref output) = tool_result {
|
if let Ok(ref output) = tool_result
|
||||||
if !output.is_empty() {
|
&& !output.is_empty()
|
||||||
let _ = self
|
{
|
||||||
.channels
|
let _ = self
|
||||||
.send_status(
|
.channels
|
||||||
&message.channel,
|
.send_status(
|
||||||
StatusUpdate::ToolResult {
|
&message.channel,
|
||||||
name: pending.tool_name.clone(),
|
StatusUpdate::ToolResult {
|
||||||
preview: truncate_for_preview(output, 200),
|
name: pending.tool_name.clone(),
|
||||||
},
|
preview: output.clone(),
|
||||||
&message.metadata,
|
},
|
||||||
)
|
&message.metadata,
|
||||||
.await;
|
)
|
||||||
}
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build context including the tool result
|
// Build context including the tool result
|
||||||
@@ -1726,15 +1731,15 @@ impl Agent {
|
|||||||
// Record result in thread
|
// Record result in thread
|
||||||
{
|
{
|
||||||
let mut sess = session.lock().await;
|
let mut sess = session.lock().await;
|
||||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
if let Some(thread) = sess.threads.get_mut(&thread_id)
|
||||||
if let Some(turn) = thread.last_turn_mut() {
|
&& let Some(turn) = thread.last_turn_mut()
|
||||||
match &tool_result {
|
{
|
||||||
Ok(output) => {
|
match &tool_result {
|
||||||
turn.record_tool_result(serde_json::json!(output));
|
Ok(output) => {
|
||||||
}
|
turn.record_tool_result(serde_json::json!(output));
|
||||||
Err(e) => {
|
}
|
||||||
turn.record_tool_error(e.to_string());
|
Err(e) => {
|
||||||
}
|
turn.record_tool_error(e.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2094,15 +2099,15 @@ impl Agent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Persist new job to database (fire-and-forget)
|
// Persist new job to database (fire-and-forget)
|
||||||
if let Some(store) = self.store() {
|
if let Some(store) = self.store()
|
||||||
if let Ok(ctx) = self.context_manager.get_context(job_id).await {
|
&& let Ok(ctx) = self.context_manager.get_context(job_id).await
|
||||||
let store = store.clone();
|
{
|
||||||
tokio::spawn(async move {
|
let store = store.clone();
|
||||||
if let Err(e) = store.save_job(&ctx).await {
|
tokio::spawn(async move {
|
||||||
tracing::warn!("Failed to persist new job {}: {}", job_id, e);
|
if let Err(e) = store.save_job(&ctx).await {
|
||||||
}
|
tracing::warn!("Failed to persist new job {}: {}", job_id, e);
|
||||||
});
|
}
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Schedule for execution
|
// Schedule for execution
|
||||||
@@ -2182,10 +2187,10 @@ impl Agent {
|
|||||||
|
|
||||||
let mut output = String::from("Jobs:\n");
|
let mut output = String::from("Jobs:\n");
|
||||||
for job_id in jobs {
|
for job_id in jobs {
|
||||||
if let Ok(ctx) = self.context_manager.get_context(job_id).await {
|
if let Ok(ctx) = self.context_manager.get_context(job_id).await
|
||||||
if ctx.user_id == user_id {
|
&& ctx.user_id == user_id
|
||||||
output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state));
|
{
|
||||||
}
|
output.push_str(&format!(" {} - {} ({:?})\n", job_id, ctx.title, ctx.state));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2636,4 +2641,70 @@ mod tests {
|
|||||||
|
|
||||||
assert!(detect_auth_awaiting("tool_activate", &result).is_none());
|
assert!(detect_auth_awaiting("tool_activate", &result).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- truncate_for_preview tests ---
|
||||||
|
|
||||||
|
use super::truncate_for_preview;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_short_input() {
|
||||||
|
assert_eq!(truncate_for_preview("hello", 10), "hello");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_empty_input() {
|
||||||
|
assert_eq!(truncate_for_preview("", 10), "");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_exact_length() {
|
||||||
|
assert_eq!(truncate_for_preview("hello", 5), "hello");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_over_limit() {
|
||||||
|
let result = truncate_for_preview("hello world, this is long", 10);
|
||||||
|
assert!(result.ends_with("..."));
|
||||||
|
// "hello worl" = 10 chars + "..."
|
||||||
|
assert_eq!(result, "hello worl...");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_collapses_newlines() {
|
||||||
|
let result = truncate_for_preview("line1\nline2\nline3", 100);
|
||||||
|
assert!(!result.contains('\n'));
|
||||||
|
assert_eq!(result, "line1 line2 line3");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_collapses_whitespace() {
|
||||||
|
let result = truncate_for_preview("hello world", 100);
|
||||||
|
assert_eq!(result, "hello world");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_multibyte_utf8() {
|
||||||
|
// Each emoji is 4 bytes. Truncating at char boundary must not panic.
|
||||||
|
let input = "😀😁😂🤣😃😄😅😆😉😊";
|
||||||
|
let result = truncate_for_preview(input, 5);
|
||||||
|
assert!(result.ends_with("..."));
|
||||||
|
// First 5 chars = 5 emoji
|
||||||
|
assert_eq!(result, "😀😁😂🤣😃...");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_cjk_characters() {
|
||||||
|
// CJK chars are 3 bytes each in UTF-8.
|
||||||
|
let input = "你好世界测试数据很长的字符串";
|
||||||
|
let result = truncate_for_preview(input, 4);
|
||||||
|
assert_eq!(result, "你好世界...");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_mixed_multibyte_and_ascii() {
|
||||||
|
let input = "hello 世界 foo";
|
||||||
|
let result = truncate_for_preview(input, 8);
|
||||||
|
// 'h','e','l','l','o',' ','世','界' = 8 chars
|
||||||
|
assert_eq!(result, "hello 世界...");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ pub mod task;
|
|||||||
pub mod undo;
|
pub mod undo;
|
||||||
pub mod worker;
|
pub mod worker;
|
||||||
|
|
||||||
|
pub(crate) use agent_loop::truncate_for_preview;
|
||||||
pub use agent_loop::{Agent, AgentDeps};
|
pub use agent_loop::{Agent, AgentDeps};
|
||||||
pub use compaction::{CompactionResult, ContextCompactor};
|
pub use compaction::{CompactionResult, ContextCompactor};
|
||||||
pub use context_monitor::{CompactionStrategy, ContextBreakdown, ContextMonitor};
|
pub use context_monitor::{CompactionStrategy, ContextBreakdown, ContextMonitor};
|
||||||
|
|||||||
@@ -24,14 +24,14 @@ use crate::agent::routine::{
|
|||||||
};
|
};
|
||||||
use crate::channels::{IncomingMessage, OutgoingResponse};
|
use crate::channels::{IncomingMessage, OutgoingResponse};
|
||||||
use crate::config::RoutineConfig;
|
use crate::config::RoutineConfig;
|
||||||
use crate::history::Store;
|
use crate::db::Database;
|
||||||
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
||||||
use crate::workspace::Workspace;
|
use crate::workspace::Workspace;
|
||||||
|
|
||||||
/// The routine execution engine.
|
/// The routine execution engine.
|
||||||
pub struct RoutineEngine {
|
pub struct RoutineEngine {
|
||||||
config: RoutineConfig,
|
config: RoutineConfig,
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
llm: Arc<dyn LlmProvider>,
|
llm: Arc<dyn LlmProvider>,
|
||||||
workspace: Arc<Workspace>,
|
workspace: Arc<Workspace>,
|
||||||
/// Sender for notifications (routed to channel manager).
|
/// Sender for notifications (routed to channel manager).
|
||||||
@@ -45,7 +45,7 @@ pub struct RoutineEngine {
|
|||||||
impl RoutineEngine {
|
impl RoutineEngine {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
config: RoutineConfig,
|
config: RoutineConfig,
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
llm: Arc<dyn LlmProvider>,
|
llm: Arc<dyn LlmProvider>,
|
||||||
workspace: Arc<Workspace>,
|
workspace: Arc<Workspace>,
|
||||||
notify_tx: mpsc::Sender<OutgoingResponse>,
|
notify_tx: mpsc::Sender<OutgoingResponse>,
|
||||||
@@ -103,10 +103,9 @@ impl RoutineEngine {
|
|||||||
if let Trigger::Event {
|
if let Trigger::Event {
|
||||||
channel: Some(ch), ..
|
channel: Some(ch), ..
|
||||||
} = &routine.trigger
|
} = &routine.trigger
|
||||||
|
&& ch != &message.channel
|
||||||
{
|
{
|
||||||
if ch != &message.channel {
|
continue;
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Regex match
|
// Regex match
|
||||||
@@ -294,7 +293,7 @@ impl RoutineEngine {
|
|||||||
|
|
||||||
/// Shared context passed to the execution function.
|
/// Shared context passed to the execution function.
|
||||||
struct EngineContext {
|
struct EngineContext {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
llm: Arc<dyn LlmProvider>,
|
llm: Arc<dyn LlmProvider>,
|
||||||
workspace: Arc<Workspace>,
|
workspace: Arc<Workspace>,
|
||||||
notify_tx: mpsc::Sender<OutgoingResponse>,
|
notify_tx: mpsc::Sender<OutgoingResponse>,
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ use crate::agent::task::{Task, TaskContext, TaskOutput};
|
|||||||
use crate::agent::worker::{Worker, WorkerDeps};
|
use crate::agent::worker::{Worker, WorkerDeps};
|
||||||
use crate::config::AgentConfig;
|
use crate::config::AgentConfig;
|
||||||
use crate::context::{ContextManager, JobContext, JobState};
|
use crate::context::{ContextManager, JobContext, JobState};
|
||||||
|
use crate::db::Database;
|
||||||
use crate::error::{Error, JobError};
|
use crate::error::{Error, JobError};
|
||||||
use crate::history::Store;
|
|
||||||
use crate::llm::LlmProvider;
|
use crate::llm::LlmProvider;
|
||||||
use crate::safety::SafetyLayer;
|
use crate::safety::SafetyLayer;
|
||||||
use crate::tools::ToolRegistry;
|
use crate::tools::ToolRegistry;
|
||||||
@@ -48,7 +48,7 @@ pub struct Scheduler {
|
|||||||
llm: Arc<dyn LlmProvider>,
|
llm: Arc<dyn LlmProvider>,
|
||||||
safety: Arc<SafetyLayer>,
|
safety: Arc<SafetyLayer>,
|
||||||
tools: Arc<ToolRegistry>,
|
tools: Arc<ToolRegistry>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
/// Running jobs (main LLM-driven jobs).
|
/// Running jobs (main LLM-driven jobs).
|
||||||
jobs: Arc<RwLock<HashMap<Uuid, ScheduledJob>>>,
|
jobs: Arc<RwLock<HashMap<Uuid, ScheduledJob>>>,
|
||||||
/// Running sub-tasks (tool executions, background tasks).
|
/// Running sub-tasks (tool executions, background tasks).
|
||||||
@@ -63,7 +63,7 @@ impl Scheduler {
|
|||||||
llm: Arc<dyn LlmProvider>,
|
llm: Arc<dyn LlmProvider>,
|
||||||
safety: Arc<SafetyLayer>,
|
safety: Arc<SafetyLayer>,
|
||||||
tools: Arc<ToolRegistry>,
|
tools: Arc<ToolRegistry>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
config,
|
config,
|
||||||
|
|||||||
+21
-21
@@ -8,8 +8,8 @@ use chrono::{DateTime, Utc};
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::context::{ContextManager, JobState};
|
use crate::context::{ContextManager, JobState};
|
||||||
|
use crate::db::Database;
|
||||||
use crate::error::RepairError;
|
use crate::error::RepairError;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::tools::{BuildRequirement, Language, SoftwareBuilder, SoftwareType, ToolRegistry};
|
use crate::tools::{BuildRequirement, Language, SoftwareBuilder, SoftwareType, ToolRegistry};
|
||||||
|
|
||||||
/// A job that has been detected as stuck.
|
/// A job that has been detected as stuck.
|
||||||
@@ -69,7 +69,7 @@ pub struct DefaultSelfRepair {
|
|||||||
#[allow(dead_code)] // Will be used for time-based stuck detection
|
#[allow(dead_code)] // Will be used for time-based stuck detection
|
||||||
stuck_threshold: Duration,
|
stuck_threshold: Duration,
|
||||||
max_repair_attempts: u32,
|
max_repair_attempts: u32,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
builder: Option<Arc<dyn SoftwareBuilder>>,
|
builder: Option<Arc<dyn SoftwareBuilder>>,
|
||||||
#[allow(dead_code)] // Will be used for tool hot-reload after repair
|
#[allow(dead_code)] // Will be used for tool hot-reload after repair
|
||||||
tools: Option<Arc<ToolRegistry>>,
|
tools: Option<Arc<ToolRegistry>>,
|
||||||
@@ -94,7 +94,7 @@ impl DefaultSelfRepair {
|
|||||||
|
|
||||||
/// Add a Store for tool failure tracking.
|
/// Add a Store for tool failure tracking.
|
||||||
#[allow(dead_code)] // Public API for configuring repair with persistence
|
#[allow(dead_code)] // Public API for configuring repair with persistence
|
||||||
pub fn with_store(mut self, store: Arc<Store>) -> Self {
|
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||||
self.store = Some(store);
|
self.store = Some(store);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
@@ -119,25 +119,25 @@ impl SelfRepair for DefaultSelfRepair {
|
|||||||
let mut stuck_jobs = Vec::new();
|
let mut stuck_jobs = Vec::new();
|
||||||
|
|
||||||
for job_id in stuck_ids {
|
for job_id in stuck_ids {
|
||||||
if let Ok(ctx) = self.context_manager.get_context(job_id).await {
|
if let Ok(ctx) = self.context_manager.get_context(job_id).await
|
||||||
if ctx.state == JobState::Stuck {
|
&& ctx.state == JobState::Stuck
|
||||||
let stuck_duration = ctx
|
{
|
||||||
.started_at
|
let stuck_duration = ctx
|
||||||
.map(|start| {
|
.started_at
|
||||||
let now = Utc::now();
|
.map(|start| {
|
||||||
let duration = now.signed_duration_since(start);
|
let now = Utc::now();
|
||||||
Duration::from_secs(duration.num_seconds().max(0) as u64)
|
let duration = now.signed_duration_since(start);
|
||||||
})
|
Duration::from_secs(duration.num_seconds().max(0) as u64)
|
||||||
.unwrap_or_default();
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
stuck_jobs.push(StuckJob {
|
stuck_jobs.push(StuckJob {
|
||||||
job_id,
|
job_id,
|
||||||
last_activity: ctx.started_at.unwrap_or(ctx.created_at),
|
last_activity: ctx.started_at.unwrap_or(ctx.created_at),
|
||||||
stuck_duration,
|
stuck_duration,
|
||||||
last_error: None,
|
last_error: None,
|
||||||
repair_attempts: ctx.repair_attempts,
|
repair_attempts: ctx.repair_attempts,
|
||||||
});
|
});
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -346,11 +346,11 @@ impl Thread {
|
|||||||
let mut turn = Turn::new(turn_number, &msg.content);
|
let mut turn = Turn::new(turn_number, &msg.content);
|
||||||
|
|
||||||
// Check if next is assistant response
|
// Check if next is assistant response
|
||||||
if let Some(next) = iter.peek() {
|
if let Some(next) = iter.peek()
|
||||||
if next.role == crate::llm::Role::Assistant {
|
&& next.role == crate::llm::Role::Assistant
|
||||||
let response = iter.next().expect("peeked");
|
{
|
||||||
turn.complete(&response.content);
|
let response = iter.next().expect("peeked");
|
||||||
}
|
turn.complete(&response.content);
|
||||||
}
|
}
|
||||||
|
|
||||||
self.turns.push(turn);
|
self.turns.push(turn);
|
||||||
|
|||||||
@@ -199,10 +199,10 @@ impl SessionManager {
|
|||||||
{
|
{
|
||||||
let sessions = self.sessions.read().await;
|
let sessions = self.sessions.read().await;
|
||||||
for user_id in &stale_users {
|
for user_id in &stale_users {
|
||||||
if let Some(session) = sessions.get(user_id) {
|
if let Some(session) = sessions.get(user_id)
|
||||||
if let Ok(sess) = session.try_lock() {
|
&& let Ok(sess) = session.try_lock()
|
||||||
stale_thread_ids.extend(sess.threads.keys());
|
{
|
||||||
}
|
stale_thread_ids.extend(sess.threads.keys());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+13
-14
@@ -93,27 +93,26 @@ impl SubmissionParser {
|
|||||||
// /thread <uuid> - switch thread
|
// /thread <uuid> - switch thread
|
||||||
if let Some(rest) = lower.strip_prefix("/thread ") {
|
if let Some(rest) = lower.strip_prefix("/thread ") {
|
||||||
let rest = rest.trim();
|
let rest = rest.trim();
|
||||||
if rest != "new" {
|
if rest != "new"
|
||||||
if let Ok(id) = Uuid::parse_str(rest) {
|
&& let Ok(id) = Uuid::parse_str(rest)
|
||||||
return Submission::SwitchThread { thread_id: id };
|
{
|
||||||
}
|
return Submission::SwitchThread { thread_id: id };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// /resume <uuid> - resume from checkpoint
|
// /resume <uuid> - resume from checkpoint
|
||||||
if let Some(rest) = lower.strip_prefix("/resume ") {
|
if let Some(rest) = lower.strip_prefix("/resume ")
|
||||||
if let Ok(id) = Uuid::parse_str(rest.trim()) {
|
&& let Ok(id) = Uuid::parse_str(rest.trim())
|
||||||
return Submission::Resume { checkpoint_id: id };
|
{
|
||||||
}
|
return Submission::Resume { checkpoint_id: id };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try structured JSON approval (from web gateway's /api/chat/approval endpoint)
|
// Try structured JSON approval (from web gateway's /api/chat/approval endpoint)
|
||||||
if trimmed.starts_with('{') {
|
if trimmed.starts_with('{')
|
||||||
if let Ok(submission) = serde_json::from_str::<Submission>(trimmed) {
|
&& let Ok(submission) = serde_json::from_str::<Submission>(trimmed)
|
||||||
if matches!(submission, Submission::ExecApproval { .. }) {
|
&& matches!(submission, Submission::ExecApproval { .. })
|
||||||
return submission;
|
{
|
||||||
}
|
return submission;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Approval responses (simple yes/no/always for pending approvals)
|
// Approval responses (simple yes/no/always for pending approvals)
|
||||||
|
|||||||
+34
-12
@@ -10,8 +10,8 @@ use uuid::Uuid;
|
|||||||
use crate::agent::scheduler::WorkerMessage;
|
use crate::agent::scheduler::WorkerMessage;
|
||||||
use crate::agent::task::TaskOutput;
|
use crate::agent::task::TaskOutput;
|
||||||
use crate::context::{ContextManager, JobState};
|
use crate::context::{ContextManager, JobState};
|
||||||
|
use crate::db::Database;
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::llm::{
|
use crate::llm::{
|
||||||
ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection,
|
ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection,
|
||||||
};
|
};
|
||||||
@@ -28,7 +28,7 @@ pub struct WorkerDeps {
|
|||||||
pub llm: Arc<dyn LlmProvider>,
|
pub llm: Arc<dyn LlmProvider>,
|
||||||
pub safety: Arc<SafetyLayer>,
|
pub safety: Arc<SafetyLayer>,
|
||||||
pub tools: Arc<ToolRegistry>,
|
pub tools: Arc<ToolRegistry>,
|
||||||
pub store: Option<Arc<Store>>,
|
pub store: Option<Arc<dyn Database>>,
|
||||||
pub timeout: Duration,
|
pub timeout: Duration,
|
||||||
pub use_planning: bool,
|
pub use_planning: bool,
|
||||||
}
|
}
|
||||||
@@ -67,7 +67,7 @@ impl Worker {
|
|||||||
&self.deps.tools
|
&self.deps.tools
|
||||||
}
|
}
|
||||||
|
|
||||||
fn store(&self) -> Option<&Arc<Store>> {
|
fn store(&self) -> Option<&Arc<dyn Database>> {
|
||||||
self.deps.store.as_ref()
|
self.deps.store.as_ref()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,11 +227,11 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check for cancellation
|
// Check for cancellation
|
||||||
if let Ok(ctx) = self.context_manager().get_context(self.job_id).await {
|
if let Ok(ctx) = self.context_manager().get_context(self.job_id).await
|
||||||
if ctx.state == JobState::Cancelled {
|
&& ctx.state == JobState::Cancelled
|
||||||
tracing::info!("Worker for job {} detected cancellation", self.job_id);
|
{
|
||||||
return Ok(());
|
tracing::info!("Worker for job {} detected cancellation", self.job_id);
|
||||||
}
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
iteration += 1;
|
iteration += 1;
|
||||||
@@ -299,6 +299,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
parameters: tc.arguments.clone(),
|
parameters: tc.arguments.clone(),
|
||||||
reasoning: String::new(),
|
reasoning: String::new(),
|
||||||
alternatives: vec![],
|
alternatives: vec![],
|
||||||
|
tool_call_id: tc.id.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
self.process_tool_result(reason_ctx, &selection, result)
|
self.process_tool_result(reason_ctx, &selection, result)
|
||||||
@@ -381,7 +382,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
tools: Arc<ToolRegistry>,
|
tools: Arc<ToolRegistry>,
|
||||||
context_manager: Arc<ContextManager>,
|
context_manager: Arc<ContextManager>,
|
||||||
safety: Arc<SafetyLayer>,
|
safety: Arc<SafetyLayer>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
job_id: Uuid,
|
job_id: Uuid,
|
||||||
tool_name: &str,
|
tool_name: &str,
|
||||||
params: &serde_json::Value,
|
params: &serde_json::Value,
|
||||||
@@ -565,7 +566,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
);
|
);
|
||||||
|
|
||||||
reason_ctx.messages.push(ChatMessage::tool_result(
|
reason_ctx.messages.push(ChatMessage::tool_result(
|
||||||
"tool_call_id",
|
&selection.tool_call_id,
|
||||||
&selection.tool_name,
|
&selection.tool_name,
|
||||||
wrapped,
|
wrapped,
|
||||||
));
|
));
|
||||||
@@ -597,7 +598,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
}
|
}
|
||||||
|
|
||||||
reason_ctx.messages.push(ChatMessage::tool_result(
|
reason_ctx.messages.push(ChatMessage::tool_result(
|
||||||
"tool_call_id",
|
&selection.tool_call_id,
|
||||||
&selection.tool_name,
|
&selection.tool_name,
|
||||||
format!("Error: {}", e),
|
format!("Error: {}", e),
|
||||||
));
|
));
|
||||||
@@ -647,12 +648,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
|||||||
.execute_tool(&action.tool_name, &action.parameters)
|
.execute_tool(&action.tool_name, &action.parameters)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
// Create a synthetic ToolSelection for process_tool_result
|
// Create a synthetic ToolSelection for process_tool_result.
|
||||||
|
// Plan actions don't originate from an LLM tool_call response so
|
||||||
|
// there is no real tool_call_id; generate a unique one.
|
||||||
let selection = ToolSelection {
|
let selection = ToolSelection {
|
||||||
tool_name: action.tool_name.clone(),
|
tool_name: action.tool_name.clone(),
|
||||||
parameters: action.parameters.clone(),
|
parameters: action.parameters.clone(),
|
||||||
reasoning: action.reasoning.clone(),
|
reasoning: action.reasoning.clone(),
|
||||||
alternatives: vec![],
|
alternatives: vec![],
|
||||||
|
tool_call_id: format!("plan_{}_{}", self.job_id, i),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Process the result
|
// Process the result
|
||||||
@@ -774,8 +778,26 @@ impl From<TaskOutput> for Result<String, Error> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use crate::llm::ToolSelection;
|
||||||
use crate::util::llm_signals_completion;
|
use crate::util::llm_signals_completion;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_tool_selection_preserves_call_id() {
|
||||||
|
let selection = ToolSelection {
|
||||||
|
tool_name: "memory_search".to_string(),
|
||||||
|
parameters: serde_json::json!({"query": "test"}),
|
||||||
|
reasoning: "Need to search memory".to_string(),
|
||||||
|
alternatives: vec![],
|
||||||
|
tool_call_id: "call_abc123".to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(selection.tool_call_id, "call_abc123");
|
||||||
|
assert_ne!(
|
||||||
|
selection.tool_call_id, "tool_call_id",
|
||||||
|
"tool_call_id must not be the hardcoded placeholder string"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_completion_positive_signals() {
|
fn test_completion_positive_signals() {
|
||||||
assert!(llm_signals_completion("The job is complete."));
|
assert!(llm_signals_completion("The job is complete."));
|
||||||
|
|||||||
+229
-166
@@ -1,147 +1,128 @@
|
|||||||
//! Bootstrap configuration for IronClaw.
|
//! Bootstrap helpers for IronClaw.
|
||||||
//!
|
//!
|
||||||
//! These are the only settings that MUST live on disk because they're needed
|
//! The only setting that truly needs disk persistence before the database is
|
||||||
//! before the database connection is established. Everything else lives in the
|
//! available is `DATABASE_URL` (chicken-and-egg: can't connect to DB without
|
||||||
//! `settings` table in PostgreSQL.
|
//! it). Everything else is auto-detected or read from env vars.
|
||||||
//!
|
//!
|
||||||
//! File: `~/.ironclaw/bootstrap.json`
|
//! File: `~/.ironclaw/.env` (standard dotenvy format)
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
/// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`.
|
||||||
|
pub fn ironclaw_env_path() -> PathBuf {
|
||||||
use crate::settings::KeySource;
|
dirs::home_dir()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("."))
|
||||||
/// Minimal config needed to connect to the database and decrypt secrets.
|
.join(".ironclaw")
|
||||||
///
|
.join(".env")
|
||||||
/// This is the only JSON file IronClaw reads from disk at startup.
|
|
||||||
/// All other configuration lives in the `settings` table in PostgreSQL.
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
||||||
pub struct BootstrapConfig {
|
|
||||||
/// Database connection URL (postgres://...).
|
|
||||||
#[serde(default)]
|
|
||||||
pub database_url: Option<String>,
|
|
||||||
|
|
||||||
/// Database connection pool size.
|
|
||||||
#[serde(default)]
|
|
||||||
pub database_pool_size: Option<usize>,
|
|
||||||
|
|
||||||
/// Source for the secrets master key.
|
|
||||||
#[serde(default)]
|
|
||||||
pub secrets_master_key_source: KeySource,
|
|
||||||
|
|
||||||
/// Whether onboarding wizard has been completed.
|
|
||||||
#[serde(default)]
|
|
||||||
pub onboard_completed: bool,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for BootstrapConfig {
|
/// Load env vars from `~/.ironclaw/.env` (in addition to the standard `.env`).
|
||||||
fn default() -> Self {
|
///
|
||||||
Self {
|
/// Call this **after** `dotenvy::dotenv()` so that the standard `./.env`
|
||||||
database_url: None,
|
/// takes priority over `~/.ironclaw/.env`. dotenvy never overwrites
|
||||||
database_pool_size: None,
|
/// existing env vars, so the effective priority is:
|
||||||
secrets_master_key_source: KeySource::None,
|
///
|
||||||
onboard_completed: false,
|
/// explicit env vars > `./.env` > `~/.ironclaw/.env`
|
||||||
}
|
///
|
||||||
|
/// If `~/.ironclaw/.env` doesn't exist but the legacy `bootstrap.json` does,
|
||||||
|
/// extracts `DATABASE_URL` from it and writes the `.env` file (one-time
|
||||||
|
/// upgrade from the old config format).
|
||||||
|
pub fn load_ironclaw_env() {
|
||||||
|
let path = ironclaw_env_path();
|
||||||
|
|
||||||
|
if !path.exists() {
|
||||||
|
// One-time upgrade: extract DATABASE_URL from legacy bootstrap.json
|
||||||
|
migrate_bootstrap_json_to_env(&path);
|
||||||
|
}
|
||||||
|
|
||||||
|
if path.exists() {
|
||||||
|
let _ = dotenvy::from_path(&path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BootstrapConfig {
|
/// If `bootstrap.json` exists, pull `database_url` out of it and write `.env`.
|
||||||
/// Default bootstrap file path: `~/.ironclaw/bootstrap.json`.
|
fn migrate_bootstrap_json_to_env(env_path: &std::path::Path) {
|
||||||
pub fn default_path() -> PathBuf {
|
let ironclaw_dir = env_path
|
||||||
dirs::home_dir()
|
.parent()
|
||||||
.unwrap_or_else(|| PathBuf::from("."))
|
.unwrap_or_else(|| std::path::Path::new("."));
|
||||||
.join(".ironclaw")
|
let bootstrap_path = ironclaw_dir.join("bootstrap.json");
|
||||||
.join("bootstrap.json")
|
|
||||||
|
if !bootstrap_path.exists() {
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Legacy settings.json path (for migration detection).
|
let content = match std::fs::read_to_string(&bootstrap_path) {
|
||||||
pub fn legacy_settings_path() -> PathBuf {
|
Ok(c) => c,
|
||||||
dirs::home_dir()
|
Err(_) => return,
|
||||||
.unwrap_or_else(|| PathBuf::from("."))
|
};
|
||||||
.join(".ironclaw")
|
|
||||||
.join("settings.json")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Load from the default path, falling back to legacy settings.json,
|
// Minimal parse: just grab database_url from the JSON
|
||||||
/// then to defaults if neither exists.
|
let parsed: serde_json::Value = match serde_json::from_str(&content) {
|
||||||
pub fn load() -> Self {
|
Ok(v) => v,
|
||||||
let bootstrap_path = Self::default_path();
|
Err(_) => return,
|
||||||
if bootstrap_path.exists() {
|
};
|
||||||
return Self::load_from(&bootstrap_path);
|
|
||||||
|
if let Some(url) = parsed.get("database_url").and_then(|v| v.as_str()) {
|
||||||
|
if let Some(parent) = env_path.parent()
|
||||||
|
&& let Err(e) = std::fs::create_dir_all(parent)
|
||||||
|
{
|
||||||
|
eprintln!("Warning: failed to create {}: {}", parent.display(), e);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
if let Err(e) = std::fs::write(env_path, format!("DATABASE_URL=\"{}\"\n", url)) {
|
||||||
// Fall back to legacy settings.json (extract just the 4 bootstrap fields)
|
eprintln!("Warning: failed to migrate bootstrap.json to .env: {}", e);
|
||||||
let legacy_path = Self::legacy_settings_path();
|
return;
|
||||||
if legacy_path.exists() {
|
|
||||||
return Self::load_from_legacy(&legacy_path);
|
|
||||||
}
|
}
|
||||||
|
rename_to_migrated(&bootstrap_path);
|
||||||
Self::default()
|
eprintln!(
|
||||||
}
|
"Migrated DATABASE_URL from bootstrap.json to {}",
|
||||||
|
env_path.display()
|
||||||
/// Load from a specific path.
|
);
|
||||||
pub fn load_from(path: &PathBuf) -> Self {
|
|
||||||
match std::fs::read_to_string(path) {
|
|
||||||
Ok(data) => serde_json::from_str(&data).unwrap_or_default(),
|
|
||||||
Err(_) => Self::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Extract bootstrap fields from a legacy settings.json.
|
|
||||||
fn load_from_legacy(path: &PathBuf) -> Self {
|
|
||||||
match std::fs::read_to_string(path) {
|
|
||||||
Ok(data) => {
|
|
||||||
// The legacy Settings struct is a superset; serde will ignore extra fields.
|
|
||||||
serde_json::from_str(&data).unwrap_or_default()
|
|
||||||
}
|
|
||||||
Err(_) => Self::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Save to the default path.
|
|
||||||
pub fn save(&self) -> std::io::Result<()> {
|
|
||||||
self.save_to(&Self::default_path())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Save to a specific path.
|
|
||||||
pub fn save_to(&self, path: &PathBuf) -> std::io::Result<()> {
|
|
||||||
if let Some(parent) = path.parent() {
|
|
||||||
std::fs::create_dir_all(parent)?;
|
|
||||||
}
|
|
||||||
let json = serde_json::to_string_pretty(self)
|
|
||||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
|
|
||||||
std::fs::write(path, json)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// One-time migration from disk config files to the database settings table.
|
/// Write `DATABASE_URL` to `~/.ironclaw/.env`.
|
||||||
///
|
///
|
||||||
/// On first boot after upgrade, checks if:
|
/// Creates the parent directory if it doesn't exist.
|
||||||
/// 1. `~/.ironclaw/settings.json` exists
|
/// The value is double-quoted so that `#` (common in URL-encoded passwords)
|
||||||
/// 2. The DB settings table is empty for this user
|
/// and other shell-special characters are preserved by dotenvy.
|
||||||
|
pub fn save_database_url(url: &str) -> std::io::Result<()> {
|
||||||
|
let path = ironclaw_env_path();
|
||||||
|
if let Some(parent) = path.parent() {
|
||||||
|
std::fs::create_dir_all(parent)?;
|
||||||
|
}
|
||||||
|
std::fs::write(&path, format!("DATABASE_URL=\"{}\"\n", url))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One-time migration of legacy `~/.ironclaw/settings.json` into the database.
|
||||||
///
|
///
|
||||||
/// If both conditions hold, migrates settings, MCP servers, and session data
|
/// Only runs when a `settings.json` exists on disk AND the DB has no settings
|
||||||
/// to the database, writes `bootstrap.json`, and renames old files to `.migrated`.
|
/// yet. After the wizard writes directly to the DB, this path is only hit by
|
||||||
|
/// users upgrading from the old disk-only configuration.
|
||||||
|
///
|
||||||
|
/// After syncing, renames `settings.json` to `.migrated` so it won't trigger again.
|
||||||
pub async fn migrate_disk_to_db(
|
pub async fn migrate_disk_to_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
) -> Result<(), MigrationError> {
|
) -> Result<(), MigrationError> {
|
||||||
let legacy_settings_path = BootstrapConfig::legacy_settings_path();
|
let ironclaw_dir = dirs::home_dir()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("."))
|
||||||
|
.join(".ironclaw");
|
||||||
|
let legacy_settings_path = ironclaw_dir.join("settings.json");
|
||||||
|
|
||||||
if !legacy_settings_path.exists() {
|
if !legacy_settings_path.exists() {
|
||||||
tracing::debug!("No legacy settings.json found, skipping disk-to-DB migration");
|
tracing::debug!("No legacy settings.json found, skipping disk-to-DB migration");
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only migrate if DB is empty for this user
|
// If DB already has settings, this is not a first boot, the wizard already
|
||||||
|
// wrote directly to the DB. Just clean up the stale file.
|
||||||
let has_settings = store.has_settings(user_id).await.map_err(|e| {
|
let has_settings = store.has_settings(user_id).await.map_err(|e| {
|
||||||
MigrationError::Database(format!("Failed to check existing settings: {}", e))
|
MigrationError::Database(format!("Failed to check existing settings: {}", e))
|
||||||
})?;
|
})?;
|
||||||
if has_settings {
|
if has_settings {
|
||||||
tracing::debug!(
|
tracing::info!("DB already has settings, renaming stale settings.json");
|
||||||
"DB already has settings for user '{}', skipping migration",
|
rename_to_migrated(&legacy_settings_path);
|
||||||
user_id
|
|
||||||
);
|
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,22 +141,14 @@ pub async fn migrate_disk_to_db(
|
|||||||
tracing::info!("Migrated {} settings to database", db_map.len());
|
tracing::info!("Migrated {} settings to database", db_map.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Write bootstrap.json with the 4 essential fields
|
// 2. Write DATABASE_URL to ~/.ironclaw/.env
|
||||||
let bootstrap = BootstrapConfig {
|
if let Some(ref url) = settings.database_url {
|
||||||
database_url: settings.database_url.clone(),
|
save_database_url(url)
|
||||||
database_pool_size: settings.database_pool_size,
|
.map_err(|e| MigrationError::Io(format!("Failed to write .env: {}", e)))?;
|
||||||
secrets_master_key_source: settings.secrets_master_key_source,
|
tracing::info!("Wrote DATABASE_URL to {}", ironclaw_env_path().display());
|
||||||
onboard_completed: settings.onboard_completed,
|
}
|
||||||
};
|
|
||||||
bootstrap
|
|
||||||
.save()
|
|
||||||
.map_err(|e| MigrationError::Io(format!("Failed to write bootstrap.json: {}", e)))?;
|
|
||||||
tracing::info!("Wrote bootstrap.json");
|
|
||||||
|
|
||||||
// 3. Migrate mcp-servers.json if it exists
|
// 3. Migrate mcp-servers.json if it exists
|
||||||
let ironclaw_dir = dirs::home_dir()
|
|
||||||
.unwrap_or_else(|| PathBuf::from("."))
|
|
||||||
.join(".ironclaw");
|
|
||||||
let mcp_path = ironclaw_dir.join("mcp-servers.json");
|
let mcp_path = ironclaw_dir.join("mcp-servers.json");
|
||||||
if mcp_path.exists() {
|
if mcp_path.exists() {
|
||||||
match std::fs::read_to_string(&mcp_path) {
|
match std::fs::read_to_string(&mcp_path) {
|
||||||
@@ -236,12 +209,19 @@ pub async fn migrate_disk_to_db(
|
|||||||
// 5. Rename settings.json to .migrated (don't delete, safety net)
|
// 5. Rename settings.json to .migrated (don't delete, safety net)
|
||||||
rename_to_migrated(&legacy_settings_path);
|
rename_to_migrated(&legacy_settings_path);
|
||||||
|
|
||||||
|
// 6. Clean up old bootstrap.json if it exists (superseded by .env)
|
||||||
|
let old_bootstrap = ironclaw_dir.join("bootstrap.json");
|
||||||
|
if old_bootstrap.exists() {
|
||||||
|
rename_to_migrated(&old_bootstrap);
|
||||||
|
tracing::info!("Renamed old bootstrap.json to .migrated");
|
||||||
|
}
|
||||||
|
|
||||||
tracing::info!("Disk-to-DB migration complete");
|
tracing::info!("Disk-to-DB migration complete");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Rename a file to `<name>.migrated` as a safety net.
|
/// Rename a file to `<name>.migrated` as a safety net.
|
||||||
fn rename_to_migrated(path: &PathBuf) {
|
fn rename_to_migrated(path: &std::path::Path) {
|
||||||
let mut migrated = path.as_os_str().to_owned();
|
let mut migrated = path.as_os_str().to_owned();
|
||||||
migrated.push(".migrated");
|
migrated.push(".migrated");
|
||||||
if let Err(e) = std::fs::rename(path, &migrated) {
|
if let Err(e) = std::fs::rename(path, &migrated) {
|
||||||
@@ -264,62 +244,145 @@ mod tests {
|
|||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_bootstrap_save_load() {
|
fn test_save_and_load_database_url() {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let path = dir.path().join("bootstrap.json");
|
let env_path = dir.path().join(".env");
|
||||||
|
|
||||||
let config = BootstrapConfig {
|
// Write in the quoted format that save_database_url uses
|
||||||
database_url: Some("postgres://localhost/test".to_string()),
|
let url = "postgres://localhost:5432/ironclaw_test";
|
||||||
database_pool_size: Some(5),
|
std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap();
|
||||||
secrets_master_key_source: KeySource::Keychain,
|
|
||||||
onboard_completed: true,
|
|
||||||
};
|
|
||||||
|
|
||||||
config.save_to(&path).unwrap();
|
// Verify the content is a valid dotenv line (quoted)
|
||||||
|
let content = std::fs::read_to_string(&env_path).unwrap();
|
||||||
let loaded = BootstrapConfig::load_from(&path);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
loaded.database_url,
|
content,
|
||||||
Some("postgres://localhost/test".to_string())
|
"DATABASE_URL=\"postgres://localhost:5432/ironclaw_test\"\n"
|
||||||
);
|
);
|
||||||
assert_eq!(loaded.database_pool_size, Some(5));
|
|
||||||
assert_eq!(loaded.secrets_master_key_source, KeySource::Keychain);
|
// Verify dotenvy can parse it (strips quotes automatically)
|
||||||
assert!(loaded.onboard_completed);
|
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(parsed.len(), 1);
|
||||||
|
assert_eq!(parsed[0].0, "DATABASE_URL");
|
||||||
|
assert_eq!(parsed[0].1, url);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_bootstrap_from_legacy_settings() {
|
fn test_save_database_url_with_hash_in_password() {
|
||||||
let dir = tempdir().unwrap();
|
let dir = tempdir().unwrap();
|
||||||
let path = dir.path().join("settings.json");
|
let env_path = dir.path().join(".env");
|
||||||
|
|
||||||
// Write a legacy settings.json with many extra fields
|
// URLs with # in the password are common (URL-encoded special chars).
|
||||||
let legacy = serde_json::json!({
|
// Without quoting, dotenvy treats # as a comment delimiter.
|
||||||
"database_url": "postgres://localhost/ironclaw",
|
let url = "postgres://user:p%23ss@localhost:5432/ironclaw";
|
||||||
"database_pool_size": 10,
|
std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap();
|
||||||
|
|
||||||
|
let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path)
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(|r| r.ok())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(parsed.len(), 1);
|
||||||
|
assert_eq!(parsed[0].0, "DATABASE_URL");
|
||||||
|
assert_eq!(parsed[0].1, url);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_save_database_url_creates_parent_dirs() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let nested = dir.path().join("deep").join("nested");
|
||||||
|
let env_path = nested.join(".env");
|
||||||
|
|
||||||
|
// Parent doesn't exist yet
|
||||||
|
assert!(!nested.exists());
|
||||||
|
|
||||||
|
// The global function uses a fixed path, so we test the logic directly
|
||||||
|
std::fs::create_dir_all(&nested).unwrap();
|
||||||
|
std::fs::write(&env_path, "DATABASE_URL=postgres://test\n").unwrap();
|
||||||
|
|
||||||
|
assert!(env_path.exists());
|
||||||
|
let content = std::fs::read_to_string(&env_path).unwrap();
|
||||||
|
assert!(content.contains("DATABASE_URL=postgres://test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ironclaw_env_path() {
|
||||||
|
let path = ironclaw_env_path();
|
||||||
|
assert!(path.ends_with(".ironclaw/.env"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_migrate_bootstrap_json_to_env() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let env_path = dir.path().join(".env");
|
||||||
|
let bootstrap_path = dir.path().join("bootstrap.json");
|
||||||
|
|
||||||
|
// Write a legacy bootstrap.json
|
||||||
|
let bootstrap_json = serde_json::json!({
|
||||||
|
"database_url": "postgres://localhost/ironclaw_upgrade",
|
||||||
|
"database_pool_size": 5,
|
||||||
"secrets_master_key_source": "keychain",
|
"secrets_master_key_source": "keychain",
|
||||||
"onboard_completed": true,
|
"onboard_completed": true
|
||||||
"selected_model": "claude-3-5-sonnet",
|
|
||||||
"agent": { "name": "testbot", "max_parallel_jobs": 3 },
|
|
||||||
"heartbeat": { "enabled": true }
|
|
||||||
});
|
});
|
||||||
std::fs::write(&path, serde_json::to_string_pretty(&legacy).unwrap()).unwrap();
|
std::fs::write(
|
||||||
|
&bootstrap_path,
|
||||||
|
serde_json::to_string_pretty(&bootstrap_json).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let config = BootstrapConfig::load_from_legacy(&path);
|
assert!(!env_path.exists());
|
||||||
|
assert!(bootstrap_path.exists());
|
||||||
|
|
||||||
|
// Run the migration
|
||||||
|
migrate_bootstrap_json_to_env(&env_path);
|
||||||
|
|
||||||
|
// .env should now exist with DATABASE_URL
|
||||||
|
assert!(env_path.exists());
|
||||||
|
let content = std::fs::read_to_string(&env_path).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
config.database_url,
|
content,
|
||||||
Some("postgres://localhost/ironclaw".to_string())
|
"DATABASE_URL=\"postgres://localhost/ironclaw_upgrade\"\n"
|
||||||
);
|
);
|
||||||
assert_eq!(config.database_pool_size, Some(10));
|
|
||||||
assert_eq!(config.secrets_master_key_source, KeySource::Keychain);
|
// bootstrap.json should be renamed to .migrated
|
||||||
assert!(config.onboard_completed);
|
assert!(!bootstrap_path.exists());
|
||||||
|
assert!(dir.path().join("bootstrap.json.migrated").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_bootstrap_defaults() {
|
fn test_migrate_bootstrap_json_no_database_url() {
|
||||||
let config = BootstrapConfig::default();
|
let dir = tempdir().unwrap();
|
||||||
assert!(config.database_url.is_none());
|
let env_path = dir.path().join(".env");
|
||||||
assert!(config.database_pool_size.is_none());
|
let bootstrap_path = dir.path().join("bootstrap.json");
|
||||||
assert_eq!(config.secrets_master_key_source, KeySource::None);
|
|
||||||
assert!(!config.onboard_completed);
|
// bootstrap.json with no database_url
|
||||||
|
let bootstrap_json = serde_json::json!({
|
||||||
|
"onboard_completed": false
|
||||||
|
});
|
||||||
|
std::fs::write(
|
||||||
|
&bootstrap_path,
|
||||||
|
serde_json::to_string_pretty(&bootstrap_json).unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
migrate_bootstrap_json_to_env(&env_path);
|
||||||
|
|
||||||
|
// .env should NOT be created
|
||||||
|
assert!(!env_path.exists());
|
||||||
|
// bootstrap.json should remain (no migration happened)
|
||||||
|
assert!(bootstrap_path.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_migrate_bootstrap_json_missing() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let env_path = dir.path().join(".env");
|
||||||
|
|
||||||
|
// No bootstrap.json at all
|
||||||
|
migrate_bootstrap_json_to_env(&env_path);
|
||||||
|
|
||||||
|
// Nothing should happen
|
||||||
|
assert!(!env_path.exists());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+17
-7
@@ -33,9 +33,16 @@ use termimad::MadSkin;
|
|||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
use tokio_stream::wrappers::ReceiverStream;
|
use tokio_stream::wrappers::ReceiverStream;
|
||||||
|
|
||||||
|
use crate::agent::truncate_for_preview;
|
||||||
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||||
use crate::error::ChannelError;
|
use crate::error::ChannelError;
|
||||||
|
|
||||||
|
/// Max characters for tool result previews in the terminal.
|
||||||
|
const CLI_TOOL_RESULT_MAX: usize = 200;
|
||||||
|
|
||||||
|
/// Max characters for thinking/status messages in the terminal.
|
||||||
|
const CLI_STATUS_MAX: usize = 200;
|
||||||
|
|
||||||
/// Slash commands available in the REPL.
|
/// Slash commands available in the REPL.
|
||||||
const SLASH_COMMANDS: &[&str] = &[
|
const SLASH_COMMANDS: &[&str] = &[
|
||||||
"/help",
|
"/help",
|
||||||
@@ -261,7 +268,7 @@ impl Channel for ReplChannel {
|
|||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
// Single message mode: send it and return
|
// Single message mode: send it and return
|
||||||
if let Some(msg) = single_message {
|
if let Some(msg) = single_message {
|
||||||
let incoming = IncomingMessage::new("repl", "user", &msg);
|
let incoming = IncomingMessage::new("repl", "default", &msg);
|
||||||
let _ = tx.blocking_send(incoming);
|
let _ = tx.blocking_send(incoming);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -329,21 +336,21 @@ impl Channel for ReplChannel {
|
|||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
|
|
||||||
let msg = IncomingMessage::new("repl", "user", line);
|
let msg = IncomingMessage::new("repl", "default", line);
|
||||||
if tx.blocking_send(msg).is_err() {
|
if tx.blocking_send(msg).is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(ReadlineError::Interrupted) => {
|
Err(ReadlineError::Interrupted) => {
|
||||||
// Ctrl+C: send /interrupt
|
// Ctrl+C: send /interrupt
|
||||||
let msg = IncomingMessage::new("repl", "user", "/interrupt");
|
let msg = IncomingMessage::new("repl", "default", "/interrupt");
|
||||||
if tx.blocking_send(msg).is_err() {
|
if tx.blocking_send(msg).is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(ReadlineError::Eof) => {
|
Err(ReadlineError::Eof) => {
|
||||||
// Ctrl+D: send /quit so the agent loop runs graceful shutdown
|
// Ctrl+D: send /quit so the agent loop runs graceful shutdown
|
||||||
let msg = IncomingMessage::new("repl", "user", "/quit");
|
let msg = IncomingMessage::new("repl", "default", "/quit");
|
||||||
let _ = tx.blocking_send(msg);
|
let _ = tx.blocking_send(msg);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -400,7 +407,8 @@ impl Channel for ReplChannel {
|
|||||||
|
|
||||||
match status {
|
match status {
|
||||||
StatusUpdate::Thinking(msg) => {
|
StatusUpdate::Thinking(msg) => {
|
||||||
eprintln!(" \x1b[90m\u{25CB} {msg}\x1b[0m");
|
let display = truncate_for_preview(&msg, CLI_STATUS_MAX);
|
||||||
|
eprintln!(" \x1b[90m\u{25CB} {display}\x1b[0m");
|
||||||
}
|
}
|
||||||
StatusUpdate::ToolStarted { name } => {
|
StatusUpdate::ToolStarted { name } => {
|
||||||
eprintln!(" \x1b[33m\u{25CB} {name}\x1b[0m");
|
eprintln!(" \x1b[33m\u{25CB} {name}\x1b[0m");
|
||||||
@@ -413,7 +421,8 @@ impl Channel for ReplChannel {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
StatusUpdate::ToolResult { name: _, preview } => {
|
StatusUpdate::ToolResult { name: _, preview } => {
|
||||||
eprintln!(" \x1b[90m{preview}\x1b[0m");
|
let display = truncate_for_preview(&preview, CLI_TOOL_RESULT_MAX);
|
||||||
|
eprintln!(" \x1b[90m{display}\x1b[0m");
|
||||||
}
|
}
|
||||||
StatusUpdate::StreamChunk(chunk) => {
|
StatusUpdate::StreamChunk(chunk) => {
|
||||||
// Print separator on the false-to-true transition
|
// Print separator on the false-to-true transition
|
||||||
@@ -438,7 +447,8 @@ impl Channel for ReplChannel {
|
|||||||
}
|
}
|
||||||
StatusUpdate::Status(msg) => {
|
StatusUpdate::Status(msg) => {
|
||||||
if debug || msg.contains("approval") || msg.contains("Approval") {
|
if debug || msg.contains("approval") || msg.contains("Approval") {
|
||||||
eprintln!(" \x1b[90m{msg}\x1b[0m");
|
let display = truncate_for_preview(&msg, CLI_STATUS_MAX);
|
||||||
|
eprintln!(" \x1b[90m{display}\x1b[0m");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
StatusUpdate::ApprovalNeeded {
|
StatusUpdate::ApprovalNeeded {
|
||||||
|
|||||||
+112
-43
@@ -76,6 +76,9 @@ struct ChannelStoreData {
|
|||||||
credentials: HashMap<String, String>,
|
credentials: HashMap<String, String>,
|
||||||
/// Pairing store for DM pairing (guest access control).
|
/// Pairing store for DM pairing (guest access control).
|
||||||
pairing_store: Arc<PairingStore>,
|
pairing_store: Arc<PairingStore>,
|
||||||
|
/// Dedicated tokio runtime for HTTP requests, lazily initialized.
|
||||||
|
/// Reused across multiple `http_request` calls within one execution.
|
||||||
|
http_runtime: Option<tokio::runtime::Runtime>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ChannelStoreData {
|
impl ChannelStoreData {
|
||||||
@@ -96,6 +99,7 @@ impl ChannelStoreData {
|
|||||||
table: ResourceTable::new(),
|
table: ResourceTable::new(),
|
||||||
credentials,
|
credentials,
|
||||||
pairing_store,
|
pairing_store,
|
||||||
|
http_runtime: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,13 +138,13 @@ impl ChannelStoreData {
|
|||||||
if result.contains('{') && result.contains('}') {
|
if result.contains('{') && result.contains('}') {
|
||||||
// Only warn if it looks like an unresolved placeholder (not JSON braces)
|
// Only warn if it looks like an unresolved placeholder (not JSON braces)
|
||||||
let brace_pattern = regex::Regex::new(r"\{[A-Z_]+\}").ok();
|
let brace_pattern = regex::Regex::new(r"\{[A-Z_]+\}").ok();
|
||||||
if let Some(re) = brace_pattern {
|
if let Some(re) = brace_pattern
|
||||||
if re.is_match(&result) {
|
&& re.is_match(&result)
|
||||||
tracing::warn!(
|
{
|
||||||
context = %context,
|
tracing::warn!(
|
||||||
"String may contain unresolved credential placeholders"
|
context = %context,
|
||||||
);
|
"String may contain unresolved credential placeholders"
|
||||||
}
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -283,10 +287,25 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
|||||||
.map(|h| h.max_response_bytes)
|
.map(|h| h.max_response_bytes)
|
||||||
.unwrap_or(10 * 1024 * 1024);
|
.unwrap_or(10 * 1024 * 1024);
|
||||||
|
|
||||||
// Make the HTTP request using blocking I/O
|
// Make the HTTP request using a dedicated single-threaded runtime.
|
||||||
// We're already in a spawn_blocking context, so we can use block_on
|
// We're inside spawn_blocking, so we can't rely on the main runtime's
|
||||||
let result = tokio::runtime::Handle::current().block_on(async {
|
// I/O driver (it may be busy with WASM compilation or other startup work).
|
||||||
let client = reqwest::Client::new();
|
// A dedicated runtime gives us our own I/O driver and avoids contention.
|
||||||
|
// The runtime is lazily created and reused across calls within one execution.
|
||||||
|
if self.http_runtime.is_none() {
|
||||||
|
self.http_runtime = Some(
|
||||||
|
tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.map_err(|e| format!("Failed to create HTTP runtime: {e}"))?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let rt = self.http_runtime.as_ref().expect("just initialized");
|
||||||
|
let result = rt.block_on(async {
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.connect_timeout(std::time::Duration::from_secs(10))
|
||||||
|
.build()
|
||||||
|
.map_err(|e| format!("Failed to build HTTP client: {e}"))?;
|
||||||
|
|
||||||
let mut request = match method.to_uppercase().as_str() {
|
let mut request = match method.to_uppercase().as_str() {
|
||||||
"GET" => client.get(&url),
|
"GET" => client.get(&url),
|
||||||
@@ -308,9 +327,9 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
|||||||
request = request.body(body_bytes);
|
request = request.body(body_bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send request with caller-specified timeout (default 30s).
|
// Send request with caller-specified timeout (default 30s, max 5min).
|
||||||
// Cap at callback_timeout to prevent outliving the host wrapper.
|
let timeout_ms = timeout_ms.unwrap_or(30_000).min(300_000) as u64;
|
||||||
let timeout = std::time::Duration::from_millis(timeout_ms.unwrap_or(30_000) as u64);
|
let timeout = std::time::Duration::from_millis(timeout_ms);
|
||||||
let response = request.timeout(timeout).send().await.map_err(|e| {
|
let response = request.timeout(timeout).send().await.map_err(|e| {
|
||||||
// Walk the full error chain so we get the actual root cause
|
// Walk the full error chain so we get the actual root cause
|
||||||
// (DNS, TLS, connection refused, etc.) instead of just
|
// (DNS, TLS, connection refused, etc.) instead of just
|
||||||
@@ -338,13 +357,13 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
|||||||
|
|
||||||
// Enforce max response body size to prevent memory exhaustion.
|
// Enforce max response body size to prevent memory exhaustion.
|
||||||
let max_response = max_response_bytes;
|
let max_response = max_response_bytes;
|
||||||
if let Some(cl) = response.content_length() {
|
if let Some(cl) = response.content_length()
|
||||||
if cl as usize > max_response {
|
&& cl as usize > max_response
|
||||||
return Err(format!(
|
{
|
||||||
"Response body too large: {} bytes exceeds limit of {} bytes",
|
return Err(format!(
|
||||||
cl, max_response
|
"Response body too large: {} bytes exceeds limit of {} bytes",
|
||||||
));
|
cl, max_response
|
||||||
}
|
));
|
||||||
}
|
}
|
||||||
let body = response
|
let body = response
|
||||||
.bytes()
|
.bytes()
|
||||||
@@ -795,7 +814,21 @@ impl WasmChannel {
|
|||||||
.await;
|
.await;
|
||||||
|
|
||||||
match result {
|
match result {
|
||||||
Ok(Ok((config, _host_state))) => {
|
Ok(Ok((config, mut host_state))) => {
|
||||||
|
// Surface WASM guest logs (errors/warnings from webhook setup, etc.)
|
||||||
|
for entry in host_state.take_logs() {
|
||||||
|
match entry.level {
|
||||||
|
crate::tools::wasm::LogLevel::Error => {
|
||||||
|
tracing::error!(channel = %self.name, "{}", entry.message);
|
||||||
|
}
|
||||||
|
crate::tools::wasm::LogLevel::Warn => {
|
||||||
|
tracing::warn!(channel = %self.name, "{}", entry.message);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
tracing::debug!(channel = %self.name, "{}", entry.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
channel = %self.name,
|
channel = %self.name,
|
||||||
display_name = %config.display_name,
|
display_name = %config.display_name,
|
||||||
@@ -1495,8 +1528,8 @@ impl WasmChannel {
|
|||||||
match result {
|
match result {
|
||||||
Ok(emitted_messages) => {
|
Ok(emitted_messages) => {
|
||||||
// Process any emitted messages
|
// Process any emitted messages
|
||||||
if !emitted_messages.is_empty() {
|
if !emitted_messages.is_empty()
|
||||||
if let Err(e) = Self::dispatch_emitted_messages(
|
&& let Err(e) = Self::dispatch_emitted_messages(
|
||||||
&channel_name,
|
&channel_name,
|
||||||
emitted_messages,
|
emitted_messages,
|
||||||
&message_tx,
|
&message_tx,
|
||||||
@@ -1508,7 +1541,6 @@ impl WasmChannel {
|
|||||||
"Failed to dispatch emitted messages from poll"
|
"Failed to dispatch emitted messages from poll"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
@@ -1738,22 +1770,22 @@ impl Channel for WasmChannel {
|
|||||||
*self.endpoints.write().await = endpoints;
|
*self.endpoints.write().await = endpoints;
|
||||||
|
|
||||||
// Start polling if configured
|
// Start polling if configured
|
||||||
if let Some(poll_config) = &config.poll {
|
if let Some(poll_config) = &config.poll
|
||||||
if poll_config.enabled {
|
&& poll_config.enabled
|
||||||
let interval = self
|
{
|
||||||
.capabilities
|
let interval = self
|
||||||
.validate_poll_interval(poll_config.interval_ms)
|
.capabilities
|
||||||
.map_err(|e| ChannelError::StartupFailed {
|
.validate_poll_interval(poll_config.interval_ms)
|
||||||
name: self.name.clone(),
|
.map_err(|e| ChannelError::StartupFailed {
|
||||||
reason: e,
|
name: self.name.clone(),
|
||||||
})?;
|
reason: e,
|
||||||
|
})?;
|
||||||
|
|
||||||
// Create shutdown channel for polling and store the sender to keep it alive
|
// Create shutdown channel for polling and store the sender to keep it alive
|
||||||
let (poll_shutdown_tx, poll_shutdown_rx) = oneshot::channel();
|
let (poll_shutdown_tx, poll_shutdown_rx) = oneshot::channel();
|
||||||
*self.poll_shutdown_tx.write().await = Some(poll_shutdown_tx);
|
*self.poll_shutdown_tx.write().await = Some(poll_shutdown_tx);
|
||||||
|
|
||||||
self.start_polling(Duration::from_millis(interval as u64), poll_shutdown_rx);
|
self.start_polling(Duration::from_millis(interval as u64), poll_shutdown_rx);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
@@ -2616,15 +2648,52 @@ mod tests {
|
|||||||
assert_eq!(store.redact_credentials(input), input);
|
assert_eq!(store.redact_credentials(input), input);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verify that the block_on-inside-spawn_blocking pattern used by the WASM
|
/// Verify that WASM HTTP host functions work using a dedicated
|
||||||
/// channel HTTP host function doesn't deadlock or panic.
|
/// current-thread runtime inside spawn_blocking.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_block_on_inside_spawn_blocking_does_not_deadlock() {
|
async fn test_dedicated_runtime_inside_spawn_blocking() {
|
||||||
let result = tokio::task::spawn_blocking(|| {
|
let result = tokio::task::spawn_blocking(|| {
|
||||||
tokio::runtime::Handle::current().block_on(async { 42 })
|
let rt = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.expect("failed to build runtime");
|
||||||
|
rt.block_on(async { 42 })
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.expect("spawn_blocking panicked");
|
.expect("spawn_blocking panicked");
|
||||||
assert_eq!(result, 42);
|
assert_eq!(result, 42);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Verify a real HTTP request works using the dedicated-runtime pattern.
|
||||||
|
/// This catches DNS, TLS, and I/O driver issues that trivial tests miss.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore] // requires network
|
||||||
|
async fn test_dedicated_runtime_real_http() {
|
||||||
|
let result = tokio::task::spawn_blocking(|| {
|
||||||
|
let rt = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.expect("failed to build runtime");
|
||||||
|
rt.block_on(async {
|
||||||
|
let client = reqwest::Client::builder()
|
||||||
|
.connect_timeout(std::time::Duration::from_secs(10))
|
||||||
|
.build()
|
||||||
|
.expect("failed to build client");
|
||||||
|
let resp = client
|
||||||
|
.get("https://api.telegram.org/bot000/getMe")
|
||||||
|
.timeout(std::time::Duration::from_secs(10))
|
||||||
|
.send()
|
||||||
|
.await;
|
||||||
|
match resp {
|
||||||
|
Ok(r) => r.status().as_u16(),
|
||||||
|
Err(e) if e.is_timeout() => panic!("request timed out: {e}"),
|
||||||
|
Err(e) => panic!("unexpected error: {e}"),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("spawn_blocking panicked");
|
||||||
|
// 404 because "000" is not a valid bot token
|
||||||
|
assert_eq!(result, 404);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-12
@@ -25,23 +25,21 @@ pub async fn auth_middleware(
|
|||||||
next: Next,
|
next: Next,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
// Try Authorization header first (constant-time comparison)
|
// Try Authorization header first (constant-time comparison)
|
||||||
if let Some(auth_header) = headers.get("authorization") {
|
if let Some(auth_header) = headers.get("authorization")
|
||||||
if let Ok(value) = auth_header.to_str() {
|
&& let Ok(value) = auth_header.to_str()
|
||||||
if let Some(token) = value.strip_prefix("Bearer ") {
|
&& let Some(token) = value.strip_prefix("Bearer ")
|
||||||
if bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) {
|
&& bool::from(token.as_bytes().ct_eq(auth.token.as_bytes()))
|
||||||
return next.run(request).await;
|
{
|
||||||
}
|
return next.run(request).await;
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fall back to query parameter for SSE EventSource (constant-time comparison)
|
// Fall back to query parameter for SSE EventSource (constant-time comparison)
|
||||||
if let Some(query) = request.uri().query() {
|
if let Some(query) = request.uri().query() {
|
||||||
for pair in query.split('&') {
|
for pair in query.split('&') {
|
||||||
if let Some(token) = pair.strip_prefix("token=") {
|
if let Some(token) = pair.strip_prefix("token=")
|
||||||
if bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) {
|
&& bool::from(token.as_bytes().ct_eq(auth.token.as_bytes()))
|
||||||
return next.run(request).await;
|
{
|
||||||
}
|
return next.run(request).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ use tokio_stream::wrappers::ReceiverStream;
|
|||||||
use crate::agent::SessionManager;
|
use crate::agent::SessionManager;
|
||||||
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||||
use crate::config::GatewayConfig;
|
use crate::config::GatewayConfig;
|
||||||
|
use crate::db::Database;
|
||||||
use crate::error::ChannelError;
|
use crate::error::ChannelError;
|
||||||
use crate::extensions::ExtensionManager;
|
use crate::extensions::ExtensionManager;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||||
use crate::tools::ToolRegistry;
|
use crate::tools::ToolRegistry;
|
||||||
use crate::workspace::Workspace;
|
use crate::workspace::Workspace;
|
||||||
@@ -147,7 +147,7 @@ impl GatewayChannel {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Inject the database store for sandbox job persistence.
|
/// Inject the database store for sandbox job persistence.
|
||||||
pub fn with_store(mut self, store: Arc<Store>) -> Self {
|
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||||
self.rebuild_state(|s| s.store = Some(store));
|
self.rebuild_state(|s| s.store = Some(store));
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -473,10 +473,10 @@ pub async fn chat_completions_handler(
|
|||||||
if let Some(mt) = req.max_tokens {
|
if let Some(mt) = req.max_tokens {
|
||||||
tool_req = tool_req.with_max_tokens(mt);
|
tool_req = tool_req.with_max_tokens(mt);
|
||||||
}
|
}
|
||||||
if let Some(ref tc) = req.tool_choice {
|
if let Some(ref tc) = req.tool_choice
|
||||||
if let Some(choice) = normalize_tool_choice(tc) {
|
&& let Some(choice) = normalize_tool_choice(tc)
|
||||||
tool_req = tool_req.with_tool_choice(choice);
|
{
|
||||||
}
|
tool_req = tool_req.with_tool_choice(choice);
|
||||||
}
|
}
|
||||||
|
|
||||||
let resp = llm
|
let resp = llm
|
||||||
@@ -591,10 +591,10 @@ async fn handle_streaming(
|
|||||||
if let Some(mt) = req.max_tokens {
|
if let Some(mt) = req.max_tokens {
|
||||||
tool_req = tool_req.with_max_tokens(mt);
|
tool_req = tool_req.with_max_tokens(mt);
|
||||||
}
|
}
|
||||||
if let Some(ref tc) = req.tool_choice {
|
if let Some(ref tc) = req.tool_choice
|
||||||
if let Some(choice) = normalize_tool_choice(tc) {
|
&& let Some(choice) = normalize_tool_choice(tc)
|
||||||
tool_req = tool_req.with_tool_choice(choice);
|
{
|
||||||
}
|
tool_req = tool_req.with_tool_choice(choice);
|
||||||
}
|
}
|
||||||
LlmResult::WithTools(
|
LlmResult::WithTools(
|
||||||
llm.complete_with_tools(tool_req)
|
llm.complete_with_tools(tool_req)
|
||||||
|
|||||||
+156
-157
@@ -30,8 +30,8 @@ use crate::channels::web::auth::{AuthState, auth_middleware};
|
|||||||
use crate::channels::web::log_layer::LogBroadcaster;
|
use crate::channels::web::log_layer::LogBroadcaster;
|
||||||
use crate::channels::web::sse::SseManager;
|
use crate::channels::web::sse::SseManager;
|
||||||
use crate::channels::web::types::*;
|
use crate::channels::web::types::*;
|
||||||
|
use crate::db::Database;
|
||||||
use crate::extensions::ExtensionManager;
|
use crate::extensions::ExtensionManager;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||||
use crate::tools::ToolRegistry;
|
use crate::tools::ToolRegistry;
|
||||||
use crate::workspace::Workspace;
|
use crate::workspace::Workspace;
|
||||||
@@ -126,7 +126,7 @@ pub struct GatewayState {
|
|||||||
/// Tool registry for listing registered tools.
|
/// Tool registry for listing registered tools.
|
||||||
pub tool_registry: Option<Arc<ToolRegistry>>,
|
pub tool_registry: Option<Arc<ToolRegistry>>,
|
||||||
/// Database store for sandbox job persistence.
|
/// Database store for sandbox job persistence.
|
||||||
pub store: Option<Arc<Store>>,
|
pub store: Option<Arc<dyn Database>>,
|
||||||
/// Container job manager for sandbox operations.
|
/// Container job manager for sandbox operations.
|
||||||
pub job_manager: Option<Arc<ContainerJobManager>>,
|
pub job_manager: Option<Arc<ContainerJobManager>>,
|
||||||
/// Prompt queue for Claude Code follow-up prompts.
|
/// Prompt queue for Claude Code follow-up prompts.
|
||||||
@@ -525,10 +525,10 @@ pub async fn clear_auth_mode(state: &GatewayState) {
|
|||||||
if let Some(ref sm) = state.session_manager {
|
if let Some(ref sm) = state.session_manager {
|
||||||
let session = sm.get_or_create_session(&state.user_id).await;
|
let session = sm.get_or_create_session(&state.user_id).await;
|
||||||
let mut sess = session.lock().await;
|
let mut sess = session.lock().await;
|
||||||
if let Some(thread_id) = sess.active_thread {
|
if let Some(thread_id) = sess.active_thread
|
||||||
if let Some(thread) = sess.threads.get_mut(&thread_id) {
|
&& let Some(thread) = sess.threads.get_mut(&thread_id)
|
||||||
thread.pending_auth = None;
|
{
|
||||||
}
|
thread.pending_auth = None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -626,69 +626,69 @@ async fn chat_history_handler(
|
|||||||
// Verify the thread belongs to the authenticated user before returning any data.
|
// Verify the thread belongs to the authenticated user before returning any data.
|
||||||
// In-memory threads are already scoped by user via session_manager, but DB
|
// In-memory threads are already scoped by user via session_manager, but DB
|
||||||
// lookups could expose another user's conversation if the UUID is guessed.
|
// lookups could expose another user's conversation if the UUID is guessed.
|
||||||
if query.thread_id.is_some() {
|
if query.thread_id.is_some()
|
||||||
if let Some(ref store) = state.store {
|
&& let Some(ref store) = state.store
|
||||||
let owned = store
|
{
|
||||||
.conversation_belongs_to_user(thread_id, &state.user_id)
|
let owned = store
|
||||||
.await
|
.conversation_belongs_to_user(thread_id, &state.user_id)
|
||||||
.unwrap_or(false);
|
.await
|
||||||
if !owned && !sess.threads.contains_key(&thread_id) {
|
.unwrap_or(false);
|
||||||
return Err((StatusCode::NOT_FOUND, "Thread not found".to_string()));
|
if !owned && !sess.threads.contains_key(&thread_id) {
|
||||||
}
|
return Err((StatusCode::NOT_FOUND, "Thread not found".to_string()));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// For paginated requests (before cursor set), always go to DB
|
// For paginated requests (before cursor set), always go to DB
|
||||||
if before_cursor.is_some() {
|
if before_cursor.is_some()
|
||||||
if let Some(ref store) = state.store {
|
&& let Some(ref store) = state.store
|
||||||
let (messages, has_more) = store
|
{
|
||||||
.list_conversation_messages_paginated(thread_id, before_cursor, limit as i64)
|
let (messages, has_more) = store
|
||||||
.await
|
.list_conversation_messages_paginated(thread_id, before_cursor, limit as i64)
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
.await
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339());
|
let oldest_timestamp = messages.first().map(|m| m.created_at.to_rfc3339());
|
||||||
let turns = build_turns_from_db_messages(&messages);
|
let turns = build_turns_from_db_messages(&messages);
|
||||||
return Ok(Json(HistoryResponse {
|
return Ok(Json(HistoryResponse {
|
||||||
thread_id,
|
thread_id,
|
||||||
turns,
|
turns,
|
||||||
has_more,
|
has_more,
|
||||||
oldest_timestamp,
|
oldest_timestamp,
|
||||||
}));
|
}));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try in-memory first (freshest data for active threads)
|
// Try in-memory first (freshest data for active threads)
|
||||||
if let Some(thread) = sess.threads.get(&thread_id) {
|
if let Some(thread) = sess.threads.get(&thread_id)
|
||||||
if !thread.turns.is_empty() {
|
&& !thread.turns.is_empty()
|
||||||
let turns: Vec<TurnInfo> = thread
|
{
|
||||||
.turns
|
let turns: Vec<TurnInfo> = thread
|
||||||
.iter()
|
.turns
|
||||||
.map(|t| TurnInfo {
|
.iter()
|
||||||
turn_number: t.turn_number,
|
.map(|t| TurnInfo {
|
||||||
user_input: t.user_input.clone(),
|
turn_number: t.turn_number,
|
||||||
response: t.response.clone(),
|
user_input: t.user_input.clone(),
|
||||||
state: format!("{:?}", t.state),
|
response: t.response.clone(),
|
||||||
started_at: t.started_at.to_rfc3339(),
|
state: format!("{:?}", t.state),
|
||||||
completed_at: t.completed_at.map(|dt| dt.to_rfc3339()),
|
started_at: t.started_at.to_rfc3339(),
|
||||||
tool_calls: t
|
completed_at: t.completed_at.map(|dt| dt.to_rfc3339()),
|
||||||
.tool_calls
|
tool_calls: t
|
||||||
.iter()
|
.tool_calls
|
||||||
.map(|tc| ToolCallInfo {
|
.iter()
|
||||||
name: tc.name.clone(),
|
.map(|tc| ToolCallInfo {
|
||||||
has_result: tc.result.is_some(),
|
name: tc.name.clone(),
|
||||||
has_error: tc.error.is_some(),
|
has_result: tc.result.is_some(),
|
||||||
})
|
has_error: tc.error.is_some(),
|
||||||
.collect(),
|
})
|
||||||
})
|
.collect(),
|
||||||
.collect();
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
return Ok(Json(HistoryResponse {
|
return Ok(Json(HistoryResponse {
|
||||||
thread_id,
|
thread_id,
|
||||||
turns,
|
turns,
|
||||||
has_more: false,
|
has_more: false,
|
||||||
oldest_timestamp: None,
|
oldest_timestamp: None,
|
||||||
}));
|
}));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fall back to DB for historical threads not in memory (paginated)
|
// Fall back to DB for historical threads not in memory (paginated)
|
||||||
@@ -738,12 +738,12 @@ fn build_turns_from_db_messages(messages: &[crate::history::ConversationMessage]
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Check if next message is an assistant response
|
// Check if next message is an assistant response
|
||||||
if let Some(next) = iter.peek() {
|
if let Some(next) = iter.peek()
|
||||||
if next.role == "assistant" {
|
&& next.role == "assistant"
|
||||||
let assistant_msg = iter.next().expect("peeked");
|
{
|
||||||
turn.response = Some(assistant_msg.content.clone());
|
let assistant_msg = iter.next().expect("peeked");
|
||||||
turn.completed_at = Some(assistant_msg.created_at.to_rfc3339());
|
turn.response = Some(assistant_msg.content.clone());
|
||||||
}
|
turn.completed_at = Some(assistant_msg.created_at.to_rfc3339());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Incomplete turn (user message without response)
|
// Incomplete turn (user message without response)
|
||||||
@@ -1126,65 +1126,65 @@ async fn jobs_detail_handler(
|
|||||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||||
|
|
||||||
// Try sandbox job from DB first, scoped to the authenticated user.
|
// Try sandbox job from DB first, scoped to the authenticated user.
|
||||||
if let Some(ref store) = state.store {
|
if let Some(ref store) = state.store
|
||||||
if let Ok(Some(job)) = store.get_sandbox_job(job_id).await {
|
&& let Ok(Some(job)) = store.get_sandbox_job(job_id).await
|
||||||
if job.user_id != state.user_id {
|
{
|
||||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
if job.user_id != state.user_id {
|
||||||
}
|
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||||
let browse_id = std::path::Path::new(&job.project_dir)
|
|
||||||
.file_name()
|
|
||||||
.map(|n| n.to_string_lossy().to_string())
|
|
||||||
.unwrap_or_else(|| job.id.to_string());
|
|
||||||
|
|
||||||
let ui_state = match job.status.as_str() {
|
|
||||||
"creating" => "pending",
|
|
||||||
"running" => "in_progress",
|
|
||||||
s => s,
|
|
||||||
};
|
|
||||||
|
|
||||||
let elapsed_secs = job.started_at.map(|start| {
|
|
||||||
let end = job.completed_at.unwrap_or_else(chrono::Utc::now);
|
|
||||||
(end - start).num_seconds().max(0) as u64
|
|
||||||
});
|
|
||||||
|
|
||||||
// Synthesize transitions from timestamps.
|
|
||||||
let mut transitions = Vec::new();
|
|
||||||
if let Some(started) = job.started_at {
|
|
||||||
transitions.push(TransitionInfo {
|
|
||||||
from: "creating".to_string(),
|
|
||||||
to: "running".to_string(),
|
|
||||||
timestamp: started.to_rfc3339(),
|
|
||||||
reason: None,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if let Some(completed) = job.completed_at {
|
|
||||||
transitions.push(TransitionInfo {
|
|
||||||
from: "running".to_string(),
|
|
||||||
to: job.status.clone(),
|
|
||||||
timestamp: completed.to_rfc3339(),
|
|
||||||
reason: job.failure_reason.clone(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return Ok(Json(JobDetailResponse {
|
|
||||||
id: job.id,
|
|
||||||
title: job.task.clone(),
|
|
||||||
description: String::new(),
|
|
||||||
state: ui_state.to_string(),
|
|
||||||
user_id: job.user_id.clone(),
|
|
||||||
created_at: job.created_at.to_rfc3339(),
|
|
||||||
started_at: job.started_at.map(|dt| dt.to_rfc3339()),
|
|
||||||
completed_at: job.completed_at.map(|dt| dt.to_rfc3339()),
|
|
||||||
elapsed_secs,
|
|
||||||
project_dir: Some(job.project_dir.clone()),
|
|
||||||
browse_url: Some(format!("/projects/{}/", browse_id)),
|
|
||||||
job_mode: {
|
|
||||||
let mode = store.get_sandbox_job_mode(job.id).await.ok().flatten();
|
|
||||||
mode.filter(|m| m != "worker")
|
|
||||||
},
|
|
||||||
transitions,
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
|
let browse_id = std::path::Path::new(&job.project_dir)
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| job.id.to_string());
|
||||||
|
|
||||||
|
let ui_state = match job.status.as_str() {
|
||||||
|
"creating" => "pending",
|
||||||
|
"running" => "in_progress",
|
||||||
|
s => s,
|
||||||
|
};
|
||||||
|
|
||||||
|
let elapsed_secs = job.started_at.map(|start| {
|
||||||
|
let end = job.completed_at.unwrap_or_else(chrono::Utc::now);
|
||||||
|
(end - start).num_seconds().max(0) as u64
|
||||||
|
});
|
||||||
|
|
||||||
|
// Synthesize transitions from timestamps.
|
||||||
|
let mut transitions = Vec::new();
|
||||||
|
if let Some(started) = job.started_at {
|
||||||
|
transitions.push(TransitionInfo {
|
||||||
|
from: "creating".to_string(),
|
||||||
|
to: "running".to_string(),
|
||||||
|
timestamp: started.to_rfc3339(),
|
||||||
|
reason: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(completed) = job.completed_at {
|
||||||
|
transitions.push(TransitionInfo {
|
||||||
|
from: "running".to_string(),
|
||||||
|
to: job.status.clone(),
|
||||||
|
timestamp: completed.to_rfc3339(),
|
||||||
|
reason: job.failure_reason.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return Ok(Json(JobDetailResponse {
|
||||||
|
id: job.id,
|
||||||
|
title: job.task.clone(),
|
||||||
|
description: String::new(),
|
||||||
|
state: ui_state.to_string(),
|
||||||
|
user_id: job.user_id.clone(),
|
||||||
|
created_at: job.created_at.to_rfc3339(),
|
||||||
|
started_at: job.started_at.map(|dt| dt.to_rfc3339()),
|
||||||
|
completed_at: job.completed_at.map(|dt| dt.to_rfc3339()),
|
||||||
|
elapsed_secs,
|
||||||
|
project_dir: Some(job.project_dir.clone()),
|
||||||
|
browse_url: Some(format!("/projects/{}/", browse_id)),
|
||||||
|
job_mode: {
|
||||||
|
let mode = store.get_sandbox_job_mode(job.id).await.ok().flatten();
|
||||||
|
mode.filter(|m| m != "worker")
|
||||||
|
},
|
||||||
|
transitions,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
Err((StatusCode::NOT_FOUND, "Job not found".to_string()))
|
Err((StatusCode::NOT_FOUND, "Job not found".to_string()))
|
||||||
@@ -1198,35 +1198,35 @@ async fn jobs_cancel_handler(
|
|||||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||||
|
|
||||||
// Try sandbox job cancellation, scoped to the authenticated user.
|
// Try sandbox job cancellation, scoped to the authenticated user.
|
||||||
if let Some(ref store) = state.store {
|
if let Some(ref store) = state.store
|
||||||
if let Ok(Some(job)) = store.get_sandbox_job(job_id).await {
|
&& let Ok(Some(job)) = store.get_sandbox_job(job_id).await
|
||||||
if job.user_id != state.user_id {
|
{
|
||||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
if job.user_id != state.user_id {
|
||||||
}
|
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||||
if job.status == "running" || job.status == "creating" {
|
|
||||||
// Stop the container if we have a job manager.
|
|
||||||
if let Some(ref jm) = state.job_manager {
|
|
||||||
if let Err(e) = jm.stop_job(job_id).await {
|
|
||||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
store
|
|
||||||
.update_sandbox_job_status(
|
|
||||||
job_id,
|
|
||||||
"failed",
|
|
||||||
Some(false),
|
|
||||||
Some("Cancelled by user"),
|
|
||||||
None,
|
|
||||||
Some(chrono::Utc::now()),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
||||||
}
|
|
||||||
return Ok(Json(serde_json::json!({
|
|
||||||
"status": "cancelled",
|
|
||||||
"job_id": job_id,
|
|
||||||
})));
|
|
||||||
}
|
}
|
||||||
|
if job.status == "running" || job.status == "creating" {
|
||||||
|
// Stop the container if we have a job manager.
|
||||||
|
if let Some(ref jm) = state.job_manager
|
||||||
|
&& let Err(e) = jm.stop_job(job_id).await
|
||||||
|
{
|
||||||
|
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation");
|
||||||
|
}
|
||||||
|
store
|
||||||
|
.update_sandbox_job_status(
|
||||||
|
job_id,
|
||||||
|
"failed",
|
||||||
|
Some(false),
|
||||||
|
Some("Cancelled by user"),
|
||||||
|
None,
|
||||||
|
Some(chrono::Utc::now()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
}
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"status": "cancelled",
|
||||||
|
"job_id": job_id,
|
||||||
|
})));
|
||||||
}
|
}
|
||||||
|
|
||||||
Err((StatusCode::NOT_FOUND, "Job not found".to_string()))
|
Err((StatusCode::NOT_FOUND, "Job not found".to_string()))
|
||||||
@@ -1334,14 +1334,13 @@ async fn jobs_prompt_handler(
|
|||||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||||
|
|
||||||
// Verify user owns this job.
|
// Verify user owns this job.
|
||||||
if let Some(ref store) = state.store {
|
if let Some(ref store) = state.store
|
||||||
if !store
|
&& !store
|
||||||
.sandbox_job_belongs_to_user(job_id, &state.user_id)
|
.sandbox_job_belongs_to_user(job_id, &state.user_id)
|
||||||
.await
|
.await
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
{
|
{
|
||||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let content = body
|
let content = body
|
||||||
|
|||||||
+50
-79
@@ -1,7 +1,9 @@
|
|||||||
//! Configuration management CLI commands.
|
//! Configuration management CLI commands.
|
||||||
//!
|
//!
|
||||||
//! Commands for viewing and modifying settings.
|
//! Commands for viewing and modifying settings.
|
||||||
//! Settings are stored in PostgreSQL (env > DB > default).
|
//! Settings are stored in the database (env > DB > default).
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use clap::Subcommand;
|
use clap::Subcommand;
|
||||||
|
|
||||||
@@ -46,11 +48,9 @@ pub enum ConfigCommand {
|
|||||||
/// Connects to the database to read/write settings. Falls back to disk
|
/// Connects to the database to read/write settings. Falls back to disk
|
||||||
/// if the database is not available.
|
/// if the database is not available.
|
||||||
pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> {
|
pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> {
|
||||||
let _ = dotenvy::dotenv();
|
|
||||||
|
|
||||||
// Try to connect to the DB for settings access
|
// Try to connect to the DB for settings access
|
||||||
let store = match connect_store().await {
|
let db: Option<Arc<dyn crate::db::Database>> = match connect_db().await {
|
||||||
Ok(s) => Some(s),
|
Ok(d) => Some(d),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"Warning: Could not connect to database ({}), using disk fallback",
|
"Warning: Could not connect to database ({}), using disk fallback",
|
||||||
@@ -60,41 +60,42 @@ pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let db_ref = db.as_deref();
|
||||||
match cmd {
|
match cmd {
|
||||||
ConfigCommand::List { filter } => list_settings(store.as_ref(), filter).await,
|
ConfigCommand::List { filter } => list_settings(db_ref, filter).await,
|
||||||
ConfigCommand::Get { path } => get_setting(store.as_ref(), &path).await,
|
ConfigCommand::Get { path } => get_setting(db_ref, &path).await,
|
||||||
ConfigCommand::Set { path, value } => set_setting(store.as_ref(), &path, &value).await,
|
ConfigCommand::Set { path, value } => set_setting(db_ref, &path, &value).await,
|
||||||
ConfigCommand::Reset { path } => reset_setting(store.as_ref(), &path).await,
|
ConfigCommand::Reset { path } => reset_setting(db_ref, &path).await,
|
||||||
ConfigCommand::Path => show_path(store.is_some()),
|
ConfigCommand::Path => show_path(db_ref.is_some()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Bootstrap a DB connection for config commands.
|
/// Bootstrap a DB connection for config commands (backend-agnostic).
|
||||||
async fn connect_store() -> anyhow::Result<crate::history::Store> {
|
async fn connect_db() -> anyhow::Result<Arc<dyn crate::db::Database>> {
|
||||||
let config = crate::config::Config::from_env()
|
let config = crate::config::Config::from_env()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
let store = crate::history::Store::new(&config.database).await?;
|
crate::db::connect_from_config(&config.database)
|
||||||
store.run_migrations().await?;
|
.await
|
||||||
Ok(store)
|
.map_err(|e| anyhow::anyhow!("{}", e))
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_USER_ID: &str = "default";
|
const DEFAULT_USER_ID: &str = "default";
|
||||||
|
|
||||||
/// Load settings: DB if available, else disk.
|
/// Load settings: DB if available, else disk.
|
||||||
async fn load_settings(store: Option<&crate::history::Store>) -> Settings {
|
async fn load_settings(store: Option<&dyn crate::db::Database>) -> Settings {
|
||||||
if let Some(store) = store {
|
if let Some(store) = store {
|
||||||
match store.get_all_settings(DEFAULT_USER_ID).await {
|
match store.get_all_settings(DEFAULT_USER_ID).await {
|
||||||
Ok(map) if !map.is_empty() => return Settings::from_db_map(&map),
|
Ok(map) if !map.is_empty() => return Settings::from_db_map(&map),
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Settings::load()
|
Settings::default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List all settings.
|
/// List all settings.
|
||||||
async fn list_settings(
|
async fn list_settings(
|
||||||
store: Option<&crate::history::Store>,
|
store: Option<&dyn crate::db::Database>,
|
||||||
filter: Option<String>,
|
filter: Option<String>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let settings = load_settings(store).await;
|
let settings = load_settings(store).await;
|
||||||
@@ -107,10 +108,10 @@ async fn list_settings(
|
|||||||
println!();
|
println!();
|
||||||
|
|
||||||
for (key, value) in all {
|
for (key, value) in all {
|
||||||
if let Some(ref f) = filter {
|
if let Some(ref f) = filter
|
||||||
if !key.starts_with(f) {
|
&& !key.starts_with(f)
|
||||||
continue;
|
{
|
||||||
}
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let display_value = if value.len() > 60 {
|
let display_value = if value.len() > 60 {
|
||||||
@@ -126,7 +127,7 @@ async fn list_settings(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get a specific setting.
|
/// Get a specific setting.
|
||||||
async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> {
|
async fn get_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> {
|
||||||
let settings = load_settings(store).await;
|
let settings = load_settings(store).await;
|
||||||
|
|
||||||
match settings.get(path) {
|
match settings.get(path) {
|
||||||
@@ -142,7 +143,7 @@ async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyho
|
|||||||
|
|
||||||
/// Set a setting value.
|
/// Set a setting value.
|
||||||
async fn set_setting(
|
async fn set_setting(
|
||||||
store: Option<&crate::history::Store>,
|
store: Option<&dyn crate::db::Database>,
|
||||||
path: &str,
|
path: &str,
|
||||||
value: &str,
|
value: &str,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
@@ -152,42 +153,36 @@ async fn set_setting(
|
|||||||
.set(path, value)
|
.set(path, value)
|
||||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
|
||||||
// Save to DB if available, otherwise disk
|
let store = store.ok_or_else(|| {
|
||||||
if let Some(store) = store {
|
anyhow::anyhow!("Database connection required to save settings. Check DATABASE_URL.")
|
||||||
let json_value = match serde_json::from_str::<serde_json::Value>(value) {
|
})?;
|
||||||
Ok(v) => v,
|
let json_value = match serde_json::from_str::<serde_json::Value>(value) {
|
||||||
Err(_) => serde_json::Value::String(value.to_string()),
|
Ok(v) => v,
|
||||||
};
|
Err(_) => serde_json::Value::String(value.to_string()),
|
||||||
store
|
};
|
||||||
.set_setting(DEFAULT_USER_ID, path, &json_value)
|
store
|
||||||
.await
|
.set_setting(DEFAULT_USER_ID, path, &json_value)
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to save to database: {}", e))?;
|
.await
|
||||||
} else {
|
.map_err(|e| anyhow::anyhow!("Failed to save to database: {}", e))?;
|
||||||
settings.save()?;
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("Set {} = {}", path, value);
|
println!("Set {} = {}", path, value);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reset a setting to default.
|
/// Reset a setting to default.
|
||||||
async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> {
|
async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> {
|
||||||
let default = Settings::default();
|
let default = Settings::default();
|
||||||
let default_value = default
|
let default_value = default
|
||||||
.get(path)
|
.get(path)
|
||||||
.ok_or_else(|| anyhow::anyhow!("Unknown setting: {}", path))?;
|
.ok_or_else(|| anyhow::anyhow!("Unknown setting: {}", path))?;
|
||||||
|
|
||||||
// Delete from DB (falling back to default) or reset on disk
|
let store = store.ok_or_else(|| {
|
||||||
if let Some(store) = store {
|
anyhow::anyhow!("Database connection required to reset settings. Check DATABASE_URL.")
|
||||||
store
|
})?;
|
||||||
.delete_setting(DEFAULT_USER_ID, path)
|
store
|
||||||
.await
|
.delete_setting(DEFAULT_USER_ID, path)
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to delete setting from database: {}", e))?;
|
.await
|
||||||
} else {
|
.map_err(|e| anyhow::anyhow!("Failed to delete setting from database: {}", e))?;
|
||||||
let mut settings = Settings::load();
|
|
||||||
settings.reset(path).map_err(|e| anyhow::anyhow!("{}", e))?;
|
|
||||||
settings.save()?;
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("Reset {} to default: {}", path, default_value);
|
println!("Reset {} to default: {}", path, default_value);
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -196,38 +191,14 @@ async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> any
|
|||||||
/// Show the settings storage info.
|
/// Show the settings storage info.
|
||||||
fn show_path(has_db: bool) -> anyhow::Result<()> {
|
fn show_path(has_db: bool) -> anyhow::Result<()> {
|
||||||
if has_db {
|
if has_db {
|
||||||
println!("Settings stored in: PostgreSQL (settings table)");
|
println!("Settings stored in: database (settings table)");
|
||||||
println!(
|
|
||||||
"Bootstrap config: {}",
|
|
||||||
crate::bootstrap::BootstrapConfig::default_path().display()
|
|
||||||
);
|
|
||||||
} else {
|
} else {
|
||||||
let path = Settings::default_path();
|
println!("Settings stored in: PostgreSQL (not connected, using defaults)");
|
||||||
println!("Settings stored in: {} (disk fallback)", path.display());
|
|
||||||
|
|
||||||
if path.exists() {
|
|
||||||
let metadata = std::fs::metadata(&path)?;
|
|
||||||
println!(" Size: {} bytes", metadata.len());
|
|
||||||
if let Ok(modified) = metadata.modified() {
|
|
||||||
use std::time::SystemTime;
|
|
||||||
let duration = SystemTime::now()
|
|
||||||
.duration_since(modified)
|
|
||||||
.unwrap_or_default();
|
|
||||||
let secs = duration.as_secs();
|
|
||||||
if secs < 60 {
|
|
||||||
println!(" Modified: {} seconds ago", secs);
|
|
||||||
} else if secs < 3600 {
|
|
||||||
println!(" Modified: {} minutes ago", secs / 60);
|
|
||||||
} else if secs < 86400 {
|
|
||||||
println!(" Modified: {} hours ago", secs / 3600);
|
|
||||||
} else {
|
|
||||||
println!(" Modified: {} days ago", secs / 86400);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
println!(" (does not exist, using defaults)");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
println!(
|
||||||
|
"Env config: {}",
|
||||||
|
crate::bootstrap::ironclaw_env_path().display()
|
||||||
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
+82
-35
@@ -8,8 +8,10 @@ use std::sync::Arc;
|
|||||||
use clap::Subcommand;
|
use clap::Subcommand;
|
||||||
|
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
use crate::history::Store;
|
use crate::db::Database;
|
||||||
use crate::secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
#[cfg(feature = "postgres")]
|
||||||
|
use crate::secrets::PostgresSecretsStore;
|
||||||
|
use crate::secrets::{SecretsCrypto, SecretsStore};
|
||||||
use crate::tools::mcp::{
|
use crate::tools::mcp::{
|
||||||
McpClient, McpServerConfig, McpSessionManager, OAuthConfig,
|
McpClient, McpServerConfig, McpSessionManager, OAuthConfig,
|
||||||
auth::{authorize_mcp_server, is_authenticated},
|
auth::{authorize_mcp_server, is_authenticated},
|
||||||
@@ -172,10 +174,10 @@ async fn add_server(
|
|||||||
config.validate()?;
|
config.validate()?;
|
||||||
|
|
||||||
// Save (DB if available, else disk)
|
// Save (DB if available, else disk)
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let mut servers = load_servers(store.as_ref()).await?;
|
let mut servers = load_servers(db.as_deref()).await?;
|
||||||
servers.upsert(config);
|
servers.upsert(config);
|
||||||
save_servers(store.as_ref(), &servers).await?;
|
save_servers(db.as_deref(), &servers).await?;
|
||||||
|
|
||||||
println!();
|
println!();
|
||||||
println!(" ✓ Added MCP server '{}'", name);
|
println!(" ✓ Added MCP server '{}'", name);
|
||||||
@@ -193,12 +195,12 @@ async fn add_server(
|
|||||||
|
|
||||||
/// Remove an MCP server.
|
/// Remove an MCP server.
|
||||||
async fn remove_server(name: String) -> anyhow::Result<()> {
|
async fn remove_server(name: String) -> anyhow::Result<()> {
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let mut servers = load_servers(store.as_ref()).await?;
|
let mut servers = load_servers(db.as_deref()).await?;
|
||||||
if !servers.remove(&name) {
|
if !servers.remove(&name) {
|
||||||
anyhow::bail!("Server '{}' not found", name);
|
anyhow::bail!("Server '{}' not found", name);
|
||||||
}
|
}
|
||||||
save_servers(store.as_ref(), &servers).await?;
|
save_servers(db.as_deref(), &servers).await?;
|
||||||
|
|
||||||
println!();
|
println!();
|
||||||
println!(" ✓ Removed MCP server '{}'", name);
|
println!(" ✓ Removed MCP server '{}'", name);
|
||||||
@@ -209,8 +211,8 @@ async fn remove_server(name: String) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
/// List configured MCP servers.
|
/// List configured MCP servers.
|
||||||
async fn list_servers(verbose: bool) -> anyhow::Result<()> {
|
async fn list_servers(verbose: bool) -> anyhow::Result<()> {
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let servers = load_servers(store.as_ref()).await?;
|
let servers = load_servers(db.as_deref()).await?;
|
||||||
|
|
||||||
if servers.servers.is_empty() {
|
if servers.servers.is_empty() {
|
||||||
println!();
|
println!();
|
||||||
@@ -268,8 +270,8 @@ async fn list_servers(verbose: bool) -> anyhow::Result<()> {
|
|||||||
/// Authenticate with an MCP server.
|
/// Authenticate with an MCP server.
|
||||||
async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> {
|
async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||||
// Get server config
|
// Get server config
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let servers = load_servers(store.as_ref()).await?;
|
let servers = load_servers(db.as_deref()).await?;
|
||||||
let server = servers
|
let server = servers
|
||||||
.get(&name)
|
.get(&name)
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -341,8 +343,8 @@ async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> {
|
|||||||
/// Test connection to an MCP server.
|
/// Test connection to an MCP server.
|
||||||
async fn test_server(name: String, user_id: String) -> anyhow::Result<()> {
|
async fn test_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||||
// Get server config
|
// Get server config
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let servers = load_servers(store.as_ref()).await?;
|
let servers = load_servers(db.as_deref()).await?;
|
||||||
let server = servers
|
let server = servers
|
||||||
.get(&name)
|
.get(&name)
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -437,8 +439,8 @@ async fn test_server(name: String, user_id: String) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
/// Toggle server enabled/disabled state.
|
/// Toggle server enabled/disabled state.
|
||||||
async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Result<()> {
|
async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Result<()> {
|
||||||
let store = connect_store().await;
|
let db = connect_db().await;
|
||||||
let mut servers = load_servers(store.as_ref()).await?;
|
let mut servers = load_servers(db.as_deref()).await?;
|
||||||
|
|
||||||
let server = servers
|
let server = servers
|
||||||
.get_mut(&name)
|
.get_mut(&name)
|
||||||
@@ -453,7 +455,7 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res
|
|||||||
};
|
};
|
||||||
|
|
||||||
server.enabled = new_state;
|
server.enabled = new_state;
|
||||||
save_servers(store.as_ref(), &servers).await?;
|
save_servers(db.as_deref(), &servers).await?;
|
||||||
|
|
||||||
let status = if new_state { "enabled" } else { "disabled" };
|
let status = if new_state { "enabled" } else { "disabled" };
|
||||||
println!();
|
println!();
|
||||||
@@ -465,18 +467,16 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res
|
|||||||
|
|
||||||
const DEFAULT_USER_ID: &str = "default";
|
const DEFAULT_USER_ID: &str = "default";
|
||||||
|
|
||||||
/// Try to connect to the database store for DB-backed config.
|
/// Try to connect to the database (backend-agnostic).
|
||||||
async fn connect_store() -> Option<Store> {
|
async fn connect_db() -> Option<Arc<dyn Database>> {
|
||||||
let config = Config::from_env().await.ok()?;
|
let config = Config::from_env().await.ok()?;
|
||||||
let store = Store::new(&config.database).await.ok()?;
|
crate::db::connect_from_config(&config.database).await.ok()
|
||||||
store.run_migrations().await.ok()?;
|
|
||||||
Some(store)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load MCP servers (DB if available, else disk).
|
/// Load MCP servers (DB if available, else disk).
|
||||||
async fn load_servers(store: Option<&Store>) -> Result<McpServersFile, config::ConfigError> {
|
async fn load_servers(db: Option<&dyn Database>) -> Result<McpServersFile, config::ConfigError> {
|
||||||
if let Some(store) = store {
|
if let Some(db) = db {
|
||||||
config::load_mcp_servers_from_db(store, DEFAULT_USER_ID).await
|
config::load_mcp_servers_from_db(db, DEFAULT_USER_ID).await
|
||||||
} else {
|
} else {
|
||||||
config::load_mcp_servers().await
|
config::load_mcp_servers().await
|
||||||
}
|
}
|
||||||
@@ -484,11 +484,11 @@ async fn load_servers(store: Option<&Store>) -> Result<McpServersFile, config::C
|
|||||||
|
|
||||||
/// Save MCP servers (DB if available, else disk).
|
/// Save MCP servers (DB if available, else disk).
|
||||||
async fn save_servers(
|
async fn save_servers(
|
||||||
store: Option<&Store>,
|
db: Option<&dyn Database>,
|
||||||
servers: &McpServersFile,
|
servers: &McpServersFile,
|
||||||
) -> Result<(), config::ConfigError> {
|
) -> Result<(), config::ConfigError> {
|
||||||
if let Some(store) = store {
|
if let Some(db) = db {
|
||||||
config::save_mcp_servers_to_db(store, DEFAULT_USER_ID, servers).await
|
config::save_mcp_servers_to_db(db, DEFAULT_USER_ID, servers).await
|
||||||
} else {
|
} else {
|
||||||
config::save_mcp_servers(servers).await
|
config::save_mcp_servers(servers).await
|
||||||
}
|
}
|
||||||
@@ -504,14 +504,61 @@ async fn get_secrets_store() -> anyhow::Result<Arc<dyn SecretsStore + Send + Syn
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let store = Store::new(&config.database).await?;
|
|
||||||
store.run_migrations().await?;
|
|
||||||
|
|
||||||
let crypto = SecretsCrypto::new(master_key.clone())?;
|
let crypto = SecretsCrypto::new(master_key.clone())?;
|
||||||
Ok(Arc::new(PostgresSecretsStore::new(
|
|
||||||
store.pool(),
|
#[cfg(feature = "postgres")]
|
||||||
Arc::new(crypto),
|
{
|
||||||
)))
|
let store = crate::history::Store::new(&config.database).await?;
|
||||||
|
store.run_migrations().await?;
|
||||||
|
Ok(Arc::new(PostgresSecretsStore::new(
|
||||||
|
store.pool(),
|
||||||
|
Arc::new(crypto),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||||
|
{
|
||||||
|
use crate::db::Database as _;
|
||||||
|
use crate::db::libsql_backend::LibSqlBackend;
|
||||||
|
use secrecy::ExposeSecret as _;
|
||||||
|
|
||||||
|
let default_path = crate::config::default_libsql_path();
|
||||||
|
let db_path = config
|
||||||
|
.database
|
||||||
|
.libsql_path
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&default_path);
|
||||||
|
|
||||||
|
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||||
|
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||||
|
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||||
|
})?;
|
||||||
|
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret())
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||||
|
} else {
|
||||||
|
LibSqlBackend::new_local(db_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||||
|
};
|
||||||
|
backend
|
||||||
|
.run_migrations()
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
|
||||||
|
return Ok(Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||||
|
backend.shared_db(),
|
||||||
|
Arc::new(crypto),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||||
|
{
|
||||||
|
let _ = crypto;
|
||||||
|
anyhow::bail!(
|
||||||
|
"No database backend available for secrets. Enable 'postgres' or 'libsql' feature."
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
+26
-1
@@ -9,6 +9,30 @@ use clap::Subcommand;
|
|||||||
|
|
||||||
use crate::workspace::{EmbeddingProvider, SearchConfig, Workspace};
|
use crate::workspace::{EmbeddingProvider, SearchConfig, Workspace};
|
||||||
|
|
||||||
|
/// Run a memory command using the Database trait (works with any backend).
|
||||||
|
pub async fn run_memory_command_with_db(
|
||||||
|
cmd: MemoryCommand,
|
||||||
|
db: std::sync::Arc<dyn crate::db::Database>,
|
||||||
|
embeddings: Option<Arc<dyn EmbeddingProvider>>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let mut workspace = Workspace::new_with_db("default", db);
|
||||||
|
if let Some(emb) = embeddings {
|
||||||
|
workspace = workspace.with_embeddings(emb);
|
||||||
|
}
|
||||||
|
|
||||||
|
match cmd {
|
||||||
|
MemoryCommand::Search { query, limit } => search(&workspace, &query, limit).await,
|
||||||
|
MemoryCommand::Read { path } => read(&workspace, &path).await,
|
||||||
|
MemoryCommand::Write {
|
||||||
|
path,
|
||||||
|
content,
|
||||||
|
append,
|
||||||
|
} => write(&workspace, &path, content, append).await,
|
||||||
|
MemoryCommand::Tree { path, depth } => tree(&workspace, &path, depth).await,
|
||||||
|
MemoryCommand::Status => status(&workspace).await,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Subcommand, Debug, Clone)]
|
#[derive(Subcommand, Debug, Clone)]
|
||||||
pub enum MemoryCommand {
|
pub enum MemoryCommand {
|
||||||
/// Search workspace memory (hybrid full-text + semantic)
|
/// Search workspace memory (hybrid full-text + semantic)
|
||||||
@@ -55,7 +79,8 @@ pub enum MemoryCommand {
|
|||||||
Status,
|
Status,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run a memory command.
|
/// Run a memory command (PostgreSQL backend).
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub async fn run_memory_command(
|
pub async fn run_memory_command(
|
||||||
cmd: MemoryCommand,
|
cmd: MemoryCommand,
|
||||||
pool: deadpool_postgres::Pool,
|
pool: deadpool_postgres::Pool,
|
||||||
|
|||||||
+5
-1
@@ -12,13 +12,17 @@
|
|||||||
mod config;
|
mod config;
|
||||||
mod mcp;
|
mod mcp;
|
||||||
pub mod memory;
|
pub mod memory;
|
||||||
|
pub mod oauth_defaults;
|
||||||
mod pairing;
|
mod pairing;
|
||||||
pub mod status;
|
pub mod status;
|
||||||
mod tool;
|
mod tool;
|
||||||
|
|
||||||
pub use config::{ConfigCommand, run_config_command};
|
pub use config::{ConfigCommand, run_config_command};
|
||||||
pub use mcp::{McpCommand, run_mcp_command};
|
pub use mcp::{McpCommand, run_mcp_command};
|
||||||
pub use memory::{MemoryCommand, run_memory_command};
|
pub use memory::MemoryCommand;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
pub use memory::run_memory_command;
|
||||||
|
pub use memory::run_memory_command_with_db;
|
||||||
pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store};
|
pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store};
|
||||||
pub use status::run_status_command;
|
pub use status::run_status_command;
|
||||||
pub use tool::{ToolCommand, run_tool_command};
|
pub use tool::{ToolCommand, run_tool_command};
|
||||||
|
|||||||
@@ -0,0 +1,343 @@
|
|||||||
|
//! Shared OAuth infrastructure: built-in credentials, callback server, landing pages.
|
||||||
|
//!
|
||||||
|
//! Every OAuth flow in the codebase (WASM tool auth, MCP server auth, NEAR AI login)
|
||||||
|
//! uses the same callback port, landing page, and listener logic from this module.
|
||||||
|
//!
|
||||||
|
//! # Built-in Credentials
|
||||||
|
//!
|
||||||
|
//! Many CLI tools (gcloud, rclone, gdrive) ship with default OAuth credentials
|
||||||
|
//! so users don't need to register their own OAuth app. Google explicitly
|
||||||
|
//! documents that client_secret for "Desktop App" / "Installed App" types
|
||||||
|
//! is NOT actually secret.
|
||||||
|
//!
|
||||||
|
//! Default credentials are hardcoded below. They can be overridden at:
|
||||||
|
//!
|
||||||
|
//! - **Compile time**: Set IRONCLAW_GOOGLE_CLIENT_ID / IRONCLAW_GOOGLE_CLIENT_SECRET
|
||||||
|
//! env vars before building to replace the hardcoded defaults.
|
||||||
|
//! - **Runtime**: Users can set GOOGLE_OAUTH_CLIENT_ID / GOOGLE_OAUTH_CLIENT_SECRET
|
||||||
|
//! env vars, which take priority over built-in defaults.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||||
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
// ── Built-in credentials ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
pub struct OAuthCredentials {
|
||||||
|
pub client_id: &'static str,
|
||||||
|
pub client_secret: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Google OAuth "Desktop App" credentials, shared across all Google tools.
|
||||||
|
/// Compile-time env vars override the hardcoded defaults below.
|
||||||
|
const GOOGLE_CLIENT_ID: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_ID") {
|
||||||
|
Some(v) => v,
|
||||||
|
None => "564604149681-efo25d43rs85v0tibdepsmdv5dsrhhr0.apps.googleusercontent.com",
|
||||||
|
};
|
||||||
|
const GOOGLE_CLIENT_SECRET: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_SECRET") {
|
||||||
|
Some(v) => v,
|
||||||
|
None => "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",
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Returns built-in OAuth credentials for a provider, keyed by secret_name.
|
||||||
|
///
|
||||||
|
/// The secret_name comes from the tool's capabilities.json `auth.secret_name` field.
|
||||||
|
/// Returns `None` if no built-in credentials are configured for that provider.
|
||||||
|
pub fn builtin_credentials(secret_name: &str) -> Option<OAuthCredentials> {
|
||||||
|
match secret_name {
|
||||||
|
"google_oauth_token" => Some(OAuthCredentials {
|
||||||
|
client_id: GOOGLE_CLIENT_ID,
|
||||||
|
client_secret: GOOGLE_CLIENT_SECRET,
|
||||||
|
}),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Shared callback server ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Fixed port for all OAuth callbacks.
|
||||||
|
///
|
||||||
|
/// Every redirect URI registered with providers must use this port:
|
||||||
|
/// `http://localhost:9876/callback` (or `/auth/callback` for NEAR AI).
|
||||||
|
pub const OAUTH_CALLBACK_PORT: u16 = 9876;
|
||||||
|
|
||||||
|
/// Error from the OAuth callback listener.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum OAuthCallbackError {
|
||||||
|
#[error("Port {0} is in use (another auth flow running?): {1}")]
|
||||||
|
PortInUse(u16, String),
|
||||||
|
|
||||||
|
#[error("Authorization denied by user")]
|
||||||
|
Denied,
|
||||||
|
|
||||||
|
#[error("Timed out waiting for authorization")]
|
||||||
|
Timeout,
|
||||||
|
|
||||||
|
#[error("IO error: {0}")]
|
||||||
|
Io(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bind the OAuth callback listener on the fixed port.
|
||||||
|
///
|
||||||
|
/// Tries IPv6 loopback (`[::1]`) first so that `http://localhost:…` redirects
|
||||||
|
/// work on systems where `localhost` resolves to `::1`. Falls back to IPv4
|
||||||
|
/// (`127.0.0.1`) only if IPv6 fails for a reason other than `AddrInUse`
|
||||||
|
/// (e.g., IPv6 not supported on the host). If the port is already occupied
|
||||||
|
/// on IPv6, the port is occupied period, so we fail immediately.
|
||||||
|
pub async fn bind_callback_listener() -> Result<TcpListener, OAuthCallbackError> {
|
||||||
|
let ipv6_addr = format!("[::1]:{}", OAUTH_CALLBACK_PORT);
|
||||||
|
match TcpListener::bind(&ipv6_addr).await {
|
||||||
|
Ok(listener) => return Ok(listener),
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||||
|
return Err(OAuthCallbackError::PortInUse(
|
||||||
|
OAUTH_CALLBACK_PORT,
|
||||||
|
e.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// IPv6 not available on this host, fall back to IPv4
|
||||||
|
}
|
||||||
|
}
|
||||||
|
TcpListener::bind(format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT))
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
if e.kind() == std::io::ErrorKind::AddrInUse {
|
||||||
|
OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string())
|
||||||
|
} else {
|
||||||
|
OAuthCallbackError::Io(e.to_string())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait for an OAuth callback and extract a query parameter value.
|
||||||
|
///
|
||||||
|
/// Listens for a GET request matching `path_prefix` (e.g., "/callback" or "/auth/callback"),
|
||||||
|
/// extracts the value of `param_name` (e.g., "code" or "token"), and shows a branded
|
||||||
|
/// landing page using `display_name` (e.g., "Google", "Notion", "NEAR AI").
|
||||||
|
///
|
||||||
|
/// Times out after 5 minutes.
|
||||||
|
pub async fn wait_for_callback(
|
||||||
|
listener: TcpListener,
|
||||||
|
path_prefix: &str,
|
||||||
|
param_name: &str,
|
||||||
|
display_name: &str,
|
||||||
|
) -> Result<String, OAuthCallbackError> {
|
||||||
|
let path_prefix = path_prefix.to_string();
|
||||||
|
let param_name = param_name.to_string();
|
||||||
|
let display_name = display_name.to_string();
|
||||||
|
|
||||||
|
tokio::time::timeout(Duration::from_secs(300), async move {
|
||||||
|
loop {
|
||||||
|
let (mut socket, _) = listener
|
||||||
|
.accept()
|
||||||
|
.await
|
||||||
|
.map_err(|e| OAuthCallbackError::Io(e.to_string()))?;
|
||||||
|
|
||||||
|
let mut reader = BufReader::new(&mut socket);
|
||||||
|
let mut request_line = String::new();
|
||||||
|
reader
|
||||||
|
.read_line(&mut request_line)
|
||||||
|
.await
|
||||||
|
.map_err(|e| OAuthCallbackError::Io(e.to_string()))?;
|
||||||
|
|
||||||
|
if let Some(path) = request_line.split_whitespace().nth(1)
|
||||||
|
&& path.starts_with(&path_prefix)
|
||||||
|
&& let Some(query) = path.split('?').nth(1)
|
||||||
|
{
|
||||||
|
// Check for error first
|
||||||
|
if query.contains("error=") {
|
||||||
|
let html = landing_html(&display_name, false);
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 400 Bad Request\r\n\
|
||||||
|
Content-Type: text/html; charset=utf-8\r\n\
|
||||||
|
Connection: close\r\n\
|
||||||
|
\r\n\
|
||||||
|
{}",
|
||||||
|
html
|
||||||
|
);
|
||||||
|
let _ = socket.write_all(response.as_bytes()).await;
|
||||||
|
return Err(OAuthCallbackError::Denied);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for the target parameter
|
||||||
|
for param in query.split('&') {
|
||||||
|
let parts: Vec<&str> = param.splitn(2, '=').collect();
|
||||||
|
if parts.len() == 2 && parts[0] == param_name {
|
||||||
|
let value = urlencoding::decode(parts[1])
|
||||||
|
.unwrap_or_else(|_| parts[1].into())
|
||||||
|
.into_owned();
|
||||||
|
|
||||||
|
let html = landing_html(&display_name, true);
|
||||||
|
let response = format!(
|
||||||
|
"HTTP/1.1 200 OK\r\n\
|
||||||
|
Content-Type: text/html; charset=utf-8\r\n\
|
||||||
|
Connection: close\r\n\
|
||||||
|
\r\n\
|
||||||
|
{}",
|
||||||
|
html
|
||||||
|
);
|
||||||
|
let _ = socket.write_all(response.as_bytes()).await;
|
||||||
|
let _ = socket.shutdown().await;
|
||||||
|
|
||||||
|
return Ok(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not the callback we're looking for
|
||||||
|
let response = "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n";
|
||||||
|
let _ = socket.write_all(response.as_bytes()).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| OAuthCallbackError::Timeout)?
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Escape a string for safe interpolation into HTML content.
|
||||||
|
fn html_escape(s: &str) -> String {
|
||||||
|
let mut out = String::with_capacity(s.len());
|
||||||
|
for c in s.chars() {
|
||||||
|
match c {
|
||||||
|
'&' => out.push_str("&"),
|
||||||
|
'<' => out.push_str("<"),
|
||||||
|
'>' => out.push_str(">"),
|
||||||
|
'"' => out.push_str("""),
|
||||||
|
'\'' => out.push_str("'"),
|
||||||
|
_ => out.push(c),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// HTML landing page shown in the browser after an OAuth redirect.
|
||||||
|
pub fn landing_html(provider_name: &str, success: bool) -> String {
|
||||||
|
let safe_name = html_escape(provider_name);
|
||||||
|
let (icon, heading, subtitle, accent) = if success {
|
||||||
|
(
|
||||||
|
r##"<div style="width:64px;height:64px;border-radius:50%;background:#22c55e;display:flex;align-items:center;justify-content:center;margin:0 auto 24px">
|
||||||
|
<svg width="32" height="32" viewBox="0 0 24 24" fill="none" stroke="#fff" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><polyline points="20 6 9 17 4 12"/></svg>
|
||||||
|
</div>"##,
|
||||||
|
format!("{} Connected", safe_name),
|
||||||
|
"You can close this window and return to your terminal.",
|
||||||
|
"#22c55e",
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
(
|
||||||
|
r##"<div style="width:64px;height:64px;border-radius:50%;background:#ef4444;display:flex;align-items:center;justify-content:center;margin:0 auto 24px">
|
||||||
|
<svg width="32" height="32" viewBox="0 0 24 24" fill="none" stroke="#fff" stroke-width="3" stroke-linecap="round" stroke-linejoin="round"><line x1="18" y1="6" x2="6" y2="18"/><line x1="6" y1="6" x2="18" y2="18"/></svg>
|
||||||
|
</div>"##,
|
||||||
|
"Authorization Failed".to_string(),
|
||||||
|
"The request was denied. You can close this window and try again.",
|
||||||
|
"#ef4444",
|
||||||
|
)
|
||||||
|
};
|
||||||
|
|
||||||
|
format!(
|
||||||
|
r#"<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||||
|
<title>IronClaw - {heading}</title>
|
||||||
|
<style>
|
||||||
|
* {{ margin:0; padding:0; box-sizing:border-box }}
|
||||||
|
body {{
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||||
|
background: #0a0a0a;
|
||||||
|
color: #e5e5e5;
|
||||||
|
display: flex;
|
||||||
|
justify-content: center;
|
||||||
|
align-items: center;
|
||||||
|
min-height: 100vh;
|
||||||
|
}}
|
||||||
|
.card {{
|
||||||
|
text-align: center;
|
||||||
|
padding: 48px 40px;
|
||||||
|
max-width: 420px;
|
||||||
|
border: 1px solid #262626;
|
||||||
|
border-radius: 16px;
|
||||||
|
background: #141414;
|
||||||
|
}}
|
||||||
|
h1 {{
|
||||||
|
font-size: 22px;
|
||||||
|
font-weight: 600;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
color: #fafafa;
|
||||||
|
}}
|
||||||
|
p {{
|
||||||
|
font-size: 14px;
|
||||||
|
color: #a3a3a3;
|
||||||
|
line-height: 1.5;
|
||||||
|
}}
|
||||||
|
.accent {{ color: {accent}; }}
|
||||||
|
.brand {{
|
||||||
|
margin-top: 32px;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #525252;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="card">
|
||||||
|
{icon}
|
||||||
|
<h1>{heading}</h1>
|
||||||
|
<p>{subtitle}</p>
|
||||||
|
<div class="brand">IronClaw</div>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>"#,
|
||||||
|
heading = heading,
|
||||||
|
icon = icon,
|
||||||
|
subtitle = subtitle,
|
||||||
|
accent = accent,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::cli::oauth_defaults::{builtin_credentials, landing_html};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_unknown_provider_returns_none() {
|
||||||
|
assert!(builtin_credentials("unknown_token").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_google_returns_based_on_compile_env() {
|
||||||
|
let creds = builtin_credentials("google_oauth_token");
|
||||||
|
assert!(creds.is_some());
|
||||||
|
let creds = creds.unwrap();
|
||||||
|
assert!(!creds.client_id.is_empty());
|
||||||
|
assert!(!creds.client_secret.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_landing_html_success_contains_key_elements() {
|
||||||
|
let html = landing_html("Google", true);
|
||||||
|
assert!(html.contains("Google Connected"));
|
||||||
|
assert!(html.contains("charset"));
|
||||||
|
assert!(html.contains("IronClaw"));
|
||||||
|
assert!(html.contains("#22c55e")); // green accent
|
||||||
|
assert!(!html.contains("Failed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_landing_html_escapes_provider_name() {
|
||||||
|
let html = landing_html("<script>alert(1)</script>", true);
|
||||||
|
assert!(!html.contains("<script>"));
|
||||||
|
assert!(html.contains("<script>"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_landing_html_error_contains_key_elements() {
|
||||||
|
let html = landing_html("Notion", false);
|
||||||
|
assert!(html.contains("Authorization Failed"));
|
||||||
|
assert!(html.contains("charset"));
|
||||||
|
assert!(html.contains("IronClaw"));
|
||||||
|
assert!(html.contains("#ef4444")); // red accent
|
||||||
|
assert!(!html.contains("Connected"));
|
||||||
|
}
|
||||||
|
}
|
||||||
+22
-17
@@ -9,7 +9,7 @@ use crate::settings::Settings;
|
|||||||
|
|
||||||
/// Run the status command, printing system health info.
|
/// Run the status command, printing system health info.
|
||||||
pub async fn run_status_command() -> anyhow::Result<()> {
|
pub async fn run_status_command() -> anyhow::Result<()> {
|
||||||
let settings = Settings::load();
|
let settings = Settings::default();
|
||||||
|
|
||||||
println!("IronClaw Status");
|
println!("IronClaw Status");
|
||||||
println!("===============\n");
|
println!("===============\n");
|
||||||
@@ -22,10 +22,9 @@ pub async fn run_status_command() -> anyhow::Result<()> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Database
|
// Database
|
||||||
let db_url_set = settings.database_url.is_some() || std::env::var("DATABASE_URL").is_ok();
|
let db_url_set = std::env::var("DATABASE_URL").is_ok();
|
||||||
print!(" Database: ");
|
print!(" Database: ");
|
||||||
if db_url_set {
|
if db_url_set {
|
||||||
// Try to connect
|
|
||||||
match check_database().await {
|
match check_database().await {
|
||||||
Ok(()) => println!("connected"),
|
Ok(()) => println!("connected"),
|
||||||
Err(e) => println!("error ({})", e),
|
Err(e) => println!("error ({})", e),
|
||||||
@@ -43,13 +42,14 @@ pub async fn run_status_command() -> anyhow::Result<()> {
|
|||||||
println!("not found (run `ironclaw onboard`)");
|
println!("not found (run `ironclaw onboard`)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Secrets
|
// Secrets (auto-detect: env var or keychain)
|
||||||
print!(" Secrets: ");
|
print!(" Secrets: ");
|
||||||
let secrets_configured = settings.secrets_master_key_source != crate::settings::KeySource::None
|
let has_env_key = std::env::var("SECRETS_MASTER_KEY").is_ok();
|
||||||
|| std::env::var("SECRETS_MASTER_KEY").is_ok()
|
let has_keychain = crate::secrets::keychain::has_master_key().await;
|
||||||
|| crate::secrets::keychain::has_master_key().await;
|
if has_env_key {
|
||||||
if secrets_configured {
|
println!("configured (env)");
|
||||||
println!("configured ({:?})", settings.secrets_master_key_source);
|
} else if has_keychain {
|
||||||
|
println!("configured (keychain)");
|
||||||
} else {
|
} else {
|
||||||
println!("not configured");
|
println!("not configured");
|
||||||
}
|
}
|
||||||
@@ -129,19 +129,18 @@ pub async fn run_status_command() -> anyhow::Result<()> {
|
|||||||
Err(_) => println!("none configured"),
|
Err(_) => println!("none configured"),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Settings path
|
// Config path
|
||||||
println!("\n Settings: {}", Settings::default_path().display());
|
println!(
|
||||||
|
"\n Config: {}",
|
||||||
|
crate::bootstrap::ironclaw_env_path().display()
|
||||||
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
async fn check_database() -> anyhow::Result<()> {
|
async fn check_database() -> anyhow::Result<()> {
|
||||||
let _ = dotenvy::dotenv();
|
let url = std::env::var("DATABASE_URL").map_err(|_| anyhow::anyhow!("DATABASE_URL not set"))?;
|
||||||
let settings = Settings::load();
|
|
||||||
let url = std::env::var("DATABASE_URL")
|
|
||||||
.ok()
|
|
||||||
.or(settings.database_url)
|
|
||||||
.ok_or_else(|| anyhow::anyhow!("no URL"))?;
|
|
||||||
|
|
||||||
let config: deadpool_postgres::Config = deadpool_postgres::Config {
|
let config: deadpool_postgres::Config = deadpool_postgres::Config {
|
||||||
url: Some(url),
|
url: Some(url),
|
||||||
@@ -167,6 +166,12 @@ async fn check_database() -> anyhow::Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "postgres"))]
|
||||||
|
async fn check_database() -> anyhow::Result<()> {
|
||||||
|
// For non-postgres backends, just report configured
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn count_wasm_files(dir: &std::path::Path) -> usize {
|
fn count_wasm_files(dir: &std::path::Path) -> usize {
|
||||||
std::fs::read_dir(dir)
|
std::fs::read_dir(dir)
|
||||||
.map(|entries| {
|
.map(|entries| {
|
||||||
|
|||||||
+235
-163
@@ -11,8 +11,11 @@ use clap::Subcommand;
|
|||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
|
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
use crate::history::Store;
|
#[allow(unused_imports)]
|
||||||
use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
use crate::db::Database;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
use crate::secrets::PostgresSecretsStore;
|
||||||
|
use crate::secrets::{CreateSecretParams, SecretsCrypto, SecretsStore};
|
||||||
use crate::tools::wasm::{CapabilitiesFile, compute_binary_hash};
|
use crate::tools::wasm::{CapabilitiesFile, compute_binary_hash};
|
||||||
|
|
||||||
/// Default tools directory.
|
/// Default tools directory.
|
||||||
@@ -420,11 +423,11 @@ async fn extract_crate_name(cargo_toml: &Path) -> anyhow::Result<String> {
|
|||||||
// Simple TOML parsing for [package] name
|
// Simple TOML parsing for [package] name
|
||||||
for line in content.lines() {
|
for line in content.lines() {
|
||||||
let line = line.trim();
|
let line = line.trim();
|
||||||
if line.starts_with("name") {
|
if line.starts_with("name")
|
||||||
if let Some((_, value)) = line.split_once('=') {
|
&& let Some((_, value)) = line.split_once('=')
|
||||||
let name = value.trim().trim_matches('"').trim_matches('\'');
|
{
|
||||||
return Ok(name.to_string());
|
let name = value.trim().trim_matches('"').trim_matches('\'');
|
||||||
}
|
return Ok(name.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,10 +491,10 @@ async fn list_tools(dir: Option<PathBuf>, verbose: bool) -> anyhow::Result<()> {
|
|||||||
|
|
||||||
if has_caps {
|
if has_caps {
|
||||||
let caps_path = path.with_extension("capabilities.json");
|
let caps_path = path.with_extension("capabilities.json");
|
||||||
if let Ok(content) = fs::read_to_string(&caps_path).await {
|
if let Ok(content) = fs::read_to_string(&caps_path).await
|
||||||
if let Ok(caps) = CapabilitiesFile::from_json(&content) {
|
&& let Ok(caps) = CapabilitiesFile::from_json(&content)
|
||||||
print_capabilities_summary(&caps);
|
{
|
||||||
}
|
print_capabilities_summary(&caps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!();
|
println!();
|
||||||
@@ -604,16 +607,16 @@ fn print_capabilities_summary(caps: &CapabilitiesFile) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref secrets) = caps.secrets {
|
if let Some(ref secrets) = caps.secrets
|
||||||
if !secrets.allowed_names.is_empty() {
|
&& !secrets.allowed_names.is_empty()
|
||||||
parts.push(format!("secrets: {}", secrets.allowed_names.len()));
|
{
|
||||||
}
|
parts.push(format!("secrets: {}", secrets.allowed_names.len()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref ws) = caps.workspace {
|
if let Some(ref ws) = caps.workspace
|
||||||
if !ws.allowed_prefixes.is_empty() {
|
&& !ws.allowed_prefixes.is_empty()
|
||||||
parts.push("workspace: read".to_string());
|
{
|
||||||
}
|
parts.push("workspace: read".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
if !parts.is_empty() {
|
if !parts.is_empty() {
|
||||||
@@ -650,30 +653,30 @@ fn print_capabilities_detail(caps: &CapabilitiesFile) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref secrets) = caps.secrets {
|
if let Some(ref secrets) = caps.secrets
|
||||||
if !secrets.allowed_names.is_empty() {
|
&& !secrets.allowed_names.is_empty()
|
||||||
println!(" Secrets (existence check only):");
|
{
|
||||||
for name in &secrets.allowed_names {
|
println!(" Secrets (existence check only):");
|
||||||
println!(" {}", name);
|
for name in &secrets.allowed_names {
|
||||||
}
|
println!(" {}", name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref tool_invoke) = caps.tool_invoke {
|
if let Some(ref tool_invoke) = caps.tool_invoke
|
||||||
if !tool_invoke.aliases.is_empty() {
|
&& !tool_invoke.aliases.is_empty()
|
||||||
println!(" Tool aliases:");
|
{
|
||||||
for (alias, real_name) in &tool_invoke.aliases {
|
println!(" Tool aliases:");
|
||||||
println!(" {} -> {}", alias, real_name);
|
for (alias, real_name) in &tool_invoke.aliases {
|
||||||
}
|
println!(" {} -> {}", alias, real_name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref ws) = caps.workspace {
|
if let Some(ref ws) = caps.workspace
|
||||||
if !ws.allowed_prefixes.is_empty() {
|
&& !ws.allowed_prefixes.is_empty()
|
||||||
println!(" Workspace read prefixes:");
|
{
|
||||||
for prefix in &ws.allowed_prefixes {
|
println!(" Workspace read prefixes:");
|
||||||
println!(" {}", prefix);
|
for prefix in &ws.allowed_prefixes {
|
||||||
}
|
println!(" {}", prefix);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -722,11 +725,58 @@ async fn auth_tool(name: String, dir: Option<PathBuf>, user_id: String) -> anyho
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let store = Store::new(&config.database).await?;
|
|
||||||
store.run_migrations().await?;
|
|
||||||
|
|
||||||
let crypto = SecretsCrypto::new(master_key.clone())?;
|
let crypto = SecretsCrypto::new(master_key.clone())?;
|
||||||
let secrets_store = Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto)));
|
|
||||||
|
let secrets_store: Arc<dyn SecretsStore + Send + Sync> = {
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
{
|
||||||
|
let store = crate::history::Store::new(&config.database).await?;
|
||||||
|
store.run_migrations().await?;
|
||||||
|
Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto)))
|
||||||
|
}
|
||||||
|
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||||
|
{
|
||||||
|
use crate::db::Database as _;
|
||||||
|
use crate::db::libsql_backend::LibSqlBackend;
|
||||||
|
use secrecy::ExposeSecret as _;
|
||||||
|
|
||||||
|
let default_path = crate::config::default_libsql_path();
|
||||||
|
let db_path = config
|
||||||
|
.database
|
||||||
|
.libsql_path
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&default_path);
|
||||||
|
|
||||||
|
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||||
|
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||||
|
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||||
|
})?;
|
||||||
|
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret())
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||||
|
} else {
|
||||||
|
LibSqlBackend::new_local(db_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||||
|
};
|
||||||
|
backend
|
||||||
|
.run_migrations()
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
|
||||||
|
Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||||
|
backend.shared_db(),
|
||||||
|
Arc::new(crypto),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||||
|
{
|
||||||
|
let _ = crypto;
|
||||||
|
anyhow::bail!(
|
||||||
|
"No database backend available for secrets. Enable 'postgres' or 'libsql' feature."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Check if already configured
|
// Check if already configured
|
||||||
let already_configured = secrets_store
|
let already_configured = secrets_store
|
||||||
@@ -752,51 +802,103 @@ async fn auth_tool(name: String, dir: Option<PathBuf>, user_id: String) -> anyho
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check for environment variable
|
// Check for environment variable
|
||||||
if let Some(ref env_var) = auth.env_var {
|
if let Some(ref env_var) = auth.env_var
|
||||||
if let Ok(token) = std::env::var(env_var) {
|
&& let Ok(token) = std::env::var(env_var)
|
||||||
if !token.is_empty() {
|
&& !token.is_empty()
|
||||||
println!(" Found {} in environment.", env_var);
|
{
|
||||||
println!();
|
println!(" Found {} in environment.", env_var);
|
||||||
|
println!();
|
||||||
|
|
||||||
// Validate if endpoint is provided
|
// Validate if endpoint is provided
|
||||||
if let Some(ref validation) = auth.validation_endpoint {
|
if let Some(ref validation) = auth.validation_endpoint {
|
||||||
print!(" Validating token...");
|
print!(" Validating token...");
|
||||||
std::io::stdout().flush()?;
|
std::io::stdout().flush()?;
|
||||||
|
|
||||||
match validate_token(&token, validation, &auth.secret_name).await {
|
match validate_token(&token, validation, &auth.secret_name).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
println!(" ✓");
|
println!(" ✓");
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
println!(" ✗");
|
println!(" ✗");
|
||||||
println!(" Validation failed: {}", e);
|
println!(" Validation failed: {}", e);
|
||||||
println!();
|
println!();
|
||||||
println!(" Falling back to manual entry...");
|
println!(" Falling back to manual entry...");
|
||||||
return auth_tool_manual(&secrets_store, &user_id, &auth).await;
|
return auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await;
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save the token
|
|
||||||
save_token(&secrets_store, &user_id, &auth, &token).await?;
|
|
||||||
print_success(display_name);
|
|
||||||
return Ok(());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Save the token
|
||||||
|
save_token(secrets_store.as_ref(), &user_id, &auth, &token, None, None).await?;
|
||||||
|
print_success(display_name);
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for OAuth configuration
|
// Check for OAuth configuration
|
||||||
if let Some(ref oauth) = auth.oauth {
|
if let Some(ref oauth) = auth.oauth {
|
||||||
return auth_tool_oauth(&secrets_store, &user_id, &auth, oauth).await;
|
// For providers with shared tokens (e.g., all Google tools share google_oauth_token),
|
||||||
|
// combine scopes from all installed tools so one auth covers everything.
|
||||||
|
let combined = combine_provider_scopes(&tools_dir, &auth.secret_name, oauth).await;
|
||||||
|
if combined.scopes.len() > oauth.scopes.len() {
|
||||||
|
let extra = combined.scopes.len() - oauth.scopes.len();
|
||||||
|
println!(
|
||||||
|
" Including scopes from {} other installed tool(s) sharing this credential.",
|
||||||
|
extra
|
||||||
|
);
|
||||||
|
println!();
|
||||||
|
}
|
||||||
|
return auth_tool_oauth(secrets_store.as_ref(), &user_id, &auth, &combined).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fall back to manual entry
|
// Fall back to manual entry
|
||||||
auth_tool_manual(&secrets_store, &user_id, &auth).await
|
auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scan the tools directory for all capabilities files sharing the same secret_name
|
||||||
|
/// and combine their OAuth scopes. This way, authing any Google tool requests scopes
|
||||||
|
/// for ALL installed Google tools, so one login covers everything.
|
||||||
|
async fn combine_provider_scopes(
|
||||||
|
tools_dir: &Path,
|
||||||
|
secret_name: &str,
|
||||||
|
base_oauth: &crate::tools::wasm::OAuthConfigSchema,
|
||||||
|
) -> crate::tools::wasm::OAuthConfigSchema {
|
||||||
|
let mut all_scopes: std::collections::HashSet<String> =
|
||||||
|
base_oauth.scopes.iter().cloned().collect();
|
||||||
|
|
||||||
|
if let Ok(mut entries) = tokio::fs::read_dir(tools_dir).await {
|
||||||
|
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||||
|
let path = entry.path();
|
||||||
|
if path.extension().and_then(|e| e.to_str()) != Some("json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let name = path
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or_default();
|
||||||
|
if !name.ends_with(".capabilities.json") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(content) = tokio::fs::read_to_string(&path).await
|
||||||
|
&& let Ok(caps) = CapabilitiesFile::from_json(&content)
|
||||||
|
&& let Some(auth) = &caps.auth
|
||||||
|
&& auth.secret_name == secret_name
|
||||||
|
&& let Some(oauth) = &auth.oauth
|
||||||
|
{
|
||||||
|
all_scopes.extend(oauth.scopes.iter().cloned());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut combined = base_oauth.clone();
|
||||||
|
combined.scopes = all_scopes.into_iter().collect();
|
||||||
|
combined.scopes.sort(); // deterministic ordering
|
||||||
|
combined
|
||||||
}
|
}
|
||||||
|
|
||||||
/// OAuth browser-based login flow.
|
/// OAuth browser-based login flow.
|
||||||
async fn auth_tool_oauth(
|
async fn auth_tool_oauth(
|
||||||
store: &PostgresSecretsStore,
|
store: &(dyn SecretsStore + Send + Sync),
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||||
oauth: &crate::tools::wasm::OAuthConfigSchema,
|
oauth: &crate::tools::wasm::OAuthConfigSchema,
|
||||||
@@ -804,12 +906,14 @@ async fn auth_tool_oauth(
|
|||||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||||
use rand::RngCore;
|
use rand::RngCore;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
|
||||||
use tokio::net::TcpListener;
|
use crate::cli::oauth_defaults::{self, OAUTH_CALLBACK_PORT};
|
||||||
|
|
||||||
let display_name = auth.display_name.as_deref().unwrap_or(&auth.secret_name);
|
let display_name = auth.display_name.as_deref().unwrap_or(&auth.secret_name);
|
||||||
|
|
||||||
// Get client_id from config or env
|
// Get client_id: capabilities file > runtime env var > built-in defaults
|
||||||
|
let builtin = oauth_defaults::builtin_credentials(&auth.secret_name);
|
||||||
|
|
||||||
let client_id = oauth
|
let client_id = oauth
|
||||||
.client_id
|
.client_id
|
||||||
.clone()
|
.clone()
|
||||||
@@ -819,41 +923,32 @@ async fn auth_tool_oauth(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|env| std::env::var(env).ok())
|
.and_then(|env| std::env::var(env).ok())
|
||||||
})
|
})
|
||||||
|
.or_else(|| builtin.as_ref().map(|c| c.client_id.to_string()))
|
||||||
.ok_or_else(|| {
|
.ok_or_else(|| {
|
||||||
anyhow::anyhow!(
|
anyhow::anyhow!(
|
||||||
"OAuth client_id not configured.\n\
|
"OAuth client_id not configured.\n\
|
||||||
Set it in the capabilities file or via environment variable."
|
Set {} env var, or build with IRONCLAW_GOOGLE_CLIENT_ID.",
|
||||||
|
oauth.client_id_env.as_deref().unwrap_or("the client_id")
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
// Get client_secret if provided
|
// Get client_secret: capabilities file > runtime env var > built-in defaults
|
||||||
let client_secret = oauth.client_secret.clone().or_else(|| {
|
let client_secret = oauth
|
||||||
oauth
|
.client_secret
|
||||||
.client_secret_env
|
.clone()
|
||||||
.as_ref()
|
.or_else(|| {
|
||||||
.and_then(|env| std::env::var(env).ok())
|
oauth
|
||||||
});
|
.client_secret_env
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|env| std::env::var(env).ok())
|
||||||
|
})
|
||||||
|
.or_else(|| builtin.as_ref().map(|c| c.client_secret.to_string()));
|
||||||
|
|
||||||
println!(" Starting OAuth authentication...");
|
println!(" Starting OAuth authentication...");
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
// Find an available port for the callback
|
let listener = oauth_defaults::bind_callback_listener().await?;
|
||||||
let mut listener = None;
|
let redirect_uri = format!("http://localhost:{}/callback", OAUTH_CALLBACK_PORT);
|
||||||
let mut port = 0;
|
|
||||||
|
|
||||||
for p in 9876..=9886 {
|
|
||||||
match TcpListener::bind(format!("127.0.0.1:{}", p)).await {
|
|
||||||
Ok(l) => {
|
|
||||||
listener = Some(l);
|
|
||||||
port = p;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(_) => continue,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let listener = listener.ok_or_else(|| anyhow::anyhow!("Could not find available port"))?;
|
|
||||||
let redirect_uri = format!("http://localhost:{}/callback", port);
|
|
||||||
|
|
||||||
// Generate PKCE verifier and challenge
|
// Generate PKCE verifier and challenge
|
||||||
let (code_verifier, code_challenge) = if oauth.use_pkce {
|
let (code_verifier, code_challenge) = if oauth.use_pkce {
|
||||||
@@ -912,65 +1007,8 @@ async fn auth_tool_oauth(
|
|||||||
|
|
||||||
println!(" Waiting for authorization...");
|
println!(" Waiting for authorization...");
|
||||||
|
|
||||||
// Wait for callback with timeout
|
let code =
|
||||||
let timeout = std::time::Duration::from_secs(300);
|
oauth_defaults::wait_for_callback(listener, "/callback", "code", display_name).await?;
|
||||||
let code = tokio::time::timeout(timeout, async {
|
|
||||||
loop {
|
|
||||||
let (mut socket, _) = listener.accept().await?;
|
|
||||||
|
|
||||||
let mut reader = BufReader::new(&mut socket);
|
|
||||||
let mut request_line = String::new();
|
|
||||||
reader.read_line(&mut request_line).await?;
|
|
||||||
|
|
||||||
// Parse GET /callback?code=xxx HTTP/1.1
|
|
||||||
if let Some(path) = request_line.split_whitespace().nth(1) {
|
|
||||||
if path.starts_with("/callback") {
|
|
||||||
if let Some(query) = path.split('?').nth(1) {
|
|
||||||
for param in query.split('&') {
|
|
||||||
let parts: Vec<&str> = param.splitn(2, '=').collect();
|
|
||||||
if parts.len() == 2 && parts[0] == "code" {
|
|
||||||
let code = urlencoding::decode(parts[1])
|
|
||||||
.unwrap_or_else(|_| parts[1].into())
|
|
||||||
.into_owned();
|
|
||||||
|
|
||||||
// Send success response
|
|
||||||
let response = format!(
|
|
||||||
"HTTP/1.1 200 OK\r\n\
|
|
||||||
Content-Type: text/html\r\n\
|
|
||||||
\r\n\
|
|
||||||
<!DOCTYPE html><html><body style=\"font-family: sans-serif; \
|
|
||||||
display: flex; justify-content: center; align-items: center; \
|
|
||||||
height: 100vh; margin: 0; background: #191919; color: white;\">\
|
|
||||||
<div style=\"text-align: center;\">\
|
|
||||||
<h1>✓ {} Connected!</h1>\
|
|
||||||
<p>You can close this window.</p>\
|
|
||||||
</div></body></html>",
|
|
||||||
display_name
|
|
||||||
);
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
let _ = socket.shutdown().await;
|
|
||||||
|
|
||||||
return Ok::<_, anyhow::Error>(code);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for error
|
|
||||||
if query.contains("error=") {
|
|
||||||
let response =
|
|
||||||
"HTTP/1.1 400 Bad Request\r\n\r\nAuthorization denied";
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
return Err(anyhow::anyhow!("Authorization denied by user"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let response = "HTTP/1.1 404 Not Found\r\n\r\n";
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|_| anyhow::anyhow!("Timed out waiting for authorization"))??;
|
|
||||||
|
|
||||||
println!();
|
println!();
|
||||||
println!(" Exchanging code for token...");
|
println!(" Exchanging code for token...");
|
||||||
@@ -1021,8 +1059,19 @@ async fn auth_tool_oauth(
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
// Save the token
|
let refresh_token = token_data.get("refresh_token").and_then(|v| v.as_str());
|
||||||
save_token(store, user_id, auth, access_token).await?;
|
let expires_in = token_data.get("expires_in").and_then(|v| v.as_u64());
|
||||||
|
|
||||||
|
// Save the token (with refresh token and expiry if provided)
|
||||||
|
save_token(
|
||||||
|
store,
|
||||||
|
user_id,
|
||||||
|
auth,
|
||||||
|
access_token,
|
||||||
|
refresh_token,
|
||||||
|
expires_in,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
// Extract any additional info for display
|
// Extract any additional info for display
|
||||||
let workspace_name = token_data
|
let workspace_name = token_data
|
||||||
@@ -1044,7 +1093,7 @@ async fn auth_tool_oauth(
|
|||||||
|
|
||||||
/// Manual token entry flow.
|
/// Manual token entry flow.
|
||||||
async fn auth_tool_manual(
|
async fn auth_tool_manual(
|
||||||
store: &PostgresSecretsStore,
|
store: &(dyn SecretsStore + Send + Sync),
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
@@ -1124,8 +1173,8 @@ async fn auth_tool_manual(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save the token
|
// Save the token (manual path: no refresh token or expiry)
|
||||||
save_token(store, user_id, auth, &token).await?;
|
save_token(store, user_id, auth, &token, None, None).await?;
|
||||||
print_success(display_name);
|
print_success(display_name);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1216,11 +1265,16 @@ async fn validate_token(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Save token to secrets store.
|
/// Save token to secrets store.
|
||||||
|
///
|
||||||
|
/// Optionally stores a refresh token (as `{secret_name}_refresh_token`) and
|
||||||
|
/// sets `expires_at` on the access token so the runtime can auto-refresh.
|
||||||
async fn save_token(
|
async fn save_token(
|
||||||
store: &PostgresSecretsStore,
|
store: &(dyn SecretsStore + Send + Sync),
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||||
token: &str,
|
token: &str,
|
||||||
|
refresh_token: Option<&str>,
|
||||||
|
expires_in: Option<u64>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let mut params = CreateSecretParams::new(&auth.secret_name, token);
|
let mut params = CreateSecretParams::new(&auth.secret_name, token);
|
||||||
|
|
||||||
@@ -1228,11 +1282,29 @@ async fn save_token(
|
|||||||
params = params.with_provider(provider);
|
params = params.with_provider(provider);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(secs) = expires_in {
|
||||||
|
let expires_at = chrono::Utc::now() + chrono::Duration::seconds(secs as i64);
|
||||||
|
params = params.with_expiry(expires_at);
|
||||||
|
}
|
||||||
|
|
||||||
store
|
store
|
||||||
.create(user_id, params)
|
.create(user_id, params)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to save token: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("Failed to save token: {}", e))?;
|
||||||
|
|
||||||
|
// Store refresh token separately (no expiry, it's long-lived)
|
||||||
|
if let Some(rt) = refresh_token {
|
||||||
|
let refresh_name = format!("{}_refresh_token", auth.secret_name);
|
||||||
|
let mut refresh_params = CreateSecretParams::new(&refresh_name, rt);
|
||||||
|
if let Some(ref provider) = auth.provider {
|
||||||
|
refresh_params = refresh_params.with_provider(provider);
|
||||||
|
}
|
||||||
|
store
|
||||||
|
.create(user_id, refresh_params)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("Failed to save refresh token: {}", e))?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+139
-67
@@ -1,9 +1,9 @@
|
|||||||
//! Configuration for IronClaw.
|
//! Configuration for IronClaw.
|
||||||
//!
|
//!
|
||||||
//! Settings are loaded with priority: env var > database > default.
|
//! Settings are loaded with priority: env var > database > default.
|
||||||
//! The database replaces the old `settings.json` file for all settings
|
//! `DATABASE_URL` lives in `~/.ironclaw/.env` (loaded via dotenvy early
|
||||||
//! except the 4 bootstrap fields (database_url, pool_size, secrets key
|
//! in startup). Everything else comes from env vars, the DB settings
|
||||||
//! source, onboard_completed) which live in `~/.ironclaw/bootstrap.json`.
|
//! table, or auto-detection.
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -38,11 +38,11 @@ impl Config {
|
|||||||
/// Priority: env var > DB settings > default.
|
/// Priority: env var > DB settings > default.
|
||||||
/// This is the primary way to load config after DB is connected.
|
/// This is the primary way to load config after DB is connected.
|
||||||
pub async fn from_db(
|
pub async fn from_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
bootstrap: &crate::bootstrap::BootstrapConfig,
|
|
||||||
) -> Result<Self, ConfigError> {
|
) -> Result<Self, ConfigError> {
|
||||||
let _ = dotenvy::dotenv();
|
let _ = dotenvy::dotenv();
|
||||||
|
crate::bootstrap::load_ironclaw_env();
|
||||||
|
|
||||||
// Load all settings from DB into a Settings struct
|
// Load all settings from DB into a Settings struct
|
||||||
let db_settings = match store.get_all_settings(user_id).await {
|
let db_settings = match store.get_all_settings(user_id).await {
|
||||||
@@ -53,7 +53,7 @@ impl Config {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Self::build(bootstrap, &db_settings).await
|
Self::build(&db_settings).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load configuration from environment variables only (no database).
|
/// Load configuration from environment variables only (no database).
|
||||||
@@ -61,20 +61,20 @@ impl Config {
|
|||||||
/// Used during early startup before the database is connected,
|
/// Used during early startup before the database is connected,
|
||||||
/// and by CLI commands that don't have DB access.
|
/// and by CLI commands that don't have DB access.
|
||||||
/// Falls back to legacy `settings.json` on disk if present.
|
/// Falls back to legacy `settings.json` on disk if present.
|
||||||
|
///
|
||||||
|
/// Loads both `./.env` (standard, higher priority) and `~/.ironclaw/.env`
|
||||||
|
/// (lower priority) via dotenvy, which never overwrites existing vars.
|
||||||
pub async fn from_env() -> Result<Self, ConfigError> {
|
pub async fn from_env() -> Result<Self, ConfigError> {
|
||||||
let _ = dotenvy::dotenv();
|
let _ = dotenvy::dotenv();
|
||||||
let bootstrap = crate::bootstrap::BootstrapConfig::load();
|
crate::bootstrap::load_ironclaw_env();
|
||||||
let settings = Settings::load();
|
let settings = Settings::load();
|
||||||
Self::build(&bootstrap, &settings).await
|
Self::build(&settings).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build config from bootstrap + settings (shared by from_env and from_db).
|
/// Build config from settings (shared by from_env and from_db).
|
||||||
async fn build(
|
async fn build(settings: &Settings) -> Result<Self, ConfigError> {
|
||||||
bootstrap: &crate::bootstrap::BootstrapConfig,
|
|
||||||
settings: &Settings,
|
|
||||||
) -> Result<Self, ConfigError> {
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
database: DatabaseConfig::resolve(bootstrap)?,
|
database: DatabaseConfig::resolve()?,
|
||||||
llm: LlmConfig::resolve(settings)?,
|
llm: LlmConfig::resolve(settings)?,
|
||||||
embeddings: EmbeddingsConfig::resolve(settings)?,
|
embeddings: EmbeddingsConfig::resolve(settings)?,
|
||||||
tunnel: TunnelConfig::resolve(settings)?,
|
tunnel: TunnelConfig::resolve(settings)?,
|
||||||
@@ -82,7 +82,7 @@ impl Config {
|
|||||||
agent: AgentConfig::resolve(settings)?,
|
agent: AgentConfig::resolve(settings)?,
|
||||||
safety: SafetyConfig::resolve()?,
|
safety: SafetyConfig::resolve()?,
|
||||||
wasm: WasmConfig::resolve()?,
|
wasm: WasmConfig::resolve()?,
|
||||||
secrets: SecretsConfig::resolve(bootstrap).await?,
|
secrets: SecretsConfig::resolve().await?,
|
||||||
builder: BuilderModeConfig::resolve()?,
|
builder: BuilderModeConfig::resolve()?,
|
||||||
heartbeat: HeartbeatConfig::resolve(settings)?,
|
heartbeat: HeartbeatConfig::resolve(settings)?,
|
||||||
routines: RoutineConfig::resolve()?,
|
routines: RoutineConfig::resolve()?,
|
||||||
@@ -107,13 +107,13 @@ impl TunnelConfig {
|
|||||||
let public_url = optional_env("TUNNEL_URL")?
|
let public_url = optional_env("TUNNEL_URL")?
|
||||||
.or_else(|| settings.tunnel.public_url.clone().filter(|s| !s.is_empty()));
|
.or_else(|| settings.tunnel.public_url.clone().filter(|s| !s.is_empty()));
|
||||||
|
|
||||||
if let Some(ref url) = public_url {
|
if let Some(ref url) = public_url
|
||||||
if !url.starts_with("https://") {
|
&& !url.starts_with("https://")
|
||||||
return Err(ConfigError::InvalidValue {
|
{
|
||||||
key: "TUNNEL_URL".to_string(),
|
return Err(ConfigError::InvalidValue {
|
||||||
message: "must start with https:// (webhooks require HTTPS)".to_string(),
|
key: "TUNNEL_URL".to_string(),
|
||||||
});
|
message: "must start with https:// (webhooks require HTTPS)".to_string(),
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self { public_url })
|
Ok(Self { public_url })
|
||||||
@@ -134,35 +134,104 @@ impl TunnelConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Which database backend to use.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
|
pub enum DatabaseBackend {
|
||||||
|
/// PostgreSQL via deadpool-postgres (default).
|
||||||
|
#[default]
|
||||||
|
Postgres,
|
||||||
|
/// libSQL/Turso embedded database.
|
||||||
|
LibSql,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::str::FromStr for DatabaseBackend {
|
||||||
|
type Err = String;
|
||||||
|
|
||||||
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||||
|
match s.to_lowercase().as_str() {
|
||||||
|
"postgres" | "postgresql" | "pg" => Ok(Self::Postgres),
|
||||||
|
"libsql" | "turso" | "sqlite" => Ok(Self::LibSql),
|
||||||
|
_ => Err(format!(
|
||||||
|
"invalid database backend '{}', expected 'postgres' or 'libsql'",
|
||||||
|
s
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Database configuration.
|
/// Database configuration.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct DatabaseConfig {
|
pub struct DatabaseConfig {
|
||||||
|
/// Which backend to use (default: Postgres).
|
||||||
|
pub backend: DatabaseBackend,
|
||||||
|
|
||||||
|
// -- PostgreSQL fields --
|
||||||
pub url: SecretString,
|
pub url: SecretString,
|
||||||
pub pool_size: usize,
|
pub pool_size: usize,
|
||||||
|
|
||||||
|
// -- libSQL fields --
|
||||||
|
/// Path to local libSQL database file (default: ~/.ironclaw/ironclaw.db).
|
||||||
|
pub libsql_path: Option<PathBuf>,
|
||||||
|
/// Turso cloud URL for remote sync (optional).
|
||||||
|
pub libsql_url: Option<String>,
|
||||||
|
/// Turso auth token (required when libsql_url is set).
|
||||||
|
pub libsql_auth_token: Option<SecretString>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DatabaseConfig {
|
impl DatabaseConfig {
|
||||||
fn resolve(bootstrap: &crate::bootstrap::BootstrapConfig) -> Result<Self, ConfigError> {
|
fn resolve() -> Result<Self, ConfigError> {
|
||||||
|
let backend: DatabaseBackend = if let Some(b) = optional_env("DATABASE_BACKEND")? {
|
||||||
|
b.parse().map_err(|e| ConfigError::InvalidValue {
|
||||||
|
key: "DATABASE_BACKEND".to_string(),
|
||||||
|
message: e,
|
||||||
|
})?
|
||||||
|
} else {
|
||||||
|
DatabaseBackend::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
// PostgreSQL URL is required only when using the postgres backend.
|
||||||
|
// For libsql backend, default to an empty placeholder.
|
||||||
|
// DATABASE_URL is loaded from ~/.ironclaw/.env via dotenvy early in startup.
|
||||||
let url = optional_env("DATABASE_URL")?
|
let url = optional_env("DATABASE_URL")?
|
||||||
.or_else(|| bootstrap.database_url.clone())
|
.or_else(|| {
|
||||||
|
if backend == DatabaseBackend::LibSql {
|
||||||
|
Some("unused://libsql".to_string())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
.ok_or_else(|| ConfigError::MissingRequired {
|
.ok_or_else(|| ConfigError::MissingRequired {
|
||||||
key: "database_url".to_string(),
|
key: "database_url".to_string(),
|
||||||
hint: "Run 'ironclaw onboard' or set DATABASE_URL environment variable".to_string(),
|
hint: "Run 'ironclaw onboard' or set DATABASE_URL environment variable".to_string(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let pool_size = optional_env("DATABASE_POOL_SIZE")?
|
let pool_size = parse_optional_env("DATABASE_POOL_SIZE", 10)?;
|
||||||
.map(|s| s.parse())
|
|
||||||
.transpose()
|
let libsql_path = optional_env("LIBSQL_PATH")?.map(PathBuf::from).or_else(|| {
|
||||||
.map_err(|e| ConfigError::InvalidValue {
|
if backend == DatabaseBackend::LibSql {
|
||||||
key: "DATABASE_POOL_SIZE".to_string(),
|
Some(default_libsql_path())
|
||||||
message: format!("must be a positive integer: {e}"),
|
} else {
|
||||||
})?
|
None
|
||||||
.or(bootstrap.database_pool_size)
|
}
|
||||||
.unwrap_or(10);
|
});
|
||||||
|
|
||||||
|
let libsql_url = optional_env("LIBSQL_URL")?;
|
||||||
|
let libsql_auth_token = optional_env("LIBSQL_AUTH_TOKEN")?.map(SecretString::from);
|
||||||
|
|
||||||
|
if libsql_url.is_some() && libsql_auth_token.is_none() {
|
||||||
|
return Err(ConfigError::MissingRequired {
|
||||||
|
key: "LIBSQL_AUTH_TOKEN".to_string(),
|
||||||
|
hint: "LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
backend,
|
||||||
url: SecretString::from(url),
|
url: SecretString::from(url),
|
||||||
pool_size,
|
pool_size,
|
||||||
|
libsql_path,
|
||||||
|
libsql_url,
|
||||||
|
libsql_auth_token,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,6 +241,14 @@ impl DatabaseConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Default libSQL database path (~/.ironclaw/ironclaw.db).
|
||||||
|
pub fn default_libsql_path() -> PathBuf {
|
||||||
|
dirs::home_dir()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("."))
|
||||||
|
.join(".ironclaw")
|
||||||
|
.join("ironclaw.db")
|
||||||
|
}
|
||||||
|
|
||||||
/// Which LLM backend to use.
|
/// Which LLM backend to use.
|
||||||
///
|
///
|
||||||
/// Defaults to `NearAi` to keep IronClaw close to the NEAR ecosystem.
|
/// Defaults to `NearAi` to keep IronClaw close to the NEAR ecosystem.
|
||||||
@@ -312,6 +389,15 @@ pub struct NearAiConfig {
|
|||||||
pub api_mode: NearAiApiMode,
|
pub api_mode: NearAiApiMode,
|
||||||
/// API key for cloud-api (required for chat_completions mode)
|
/// API key for cloud-api (required for chat_completions mode)
|
||||||
pub api_key: Option<SecretString>,
|
pub api_key: Option<SecretString>,
|
||||||
|
/// Optional fallback model for failover (default: None).
|
||||||
|
/// When set, a secondary provider is created with this model and wrapped
|
||||||
|
/// in a `FailoverProvider` so transient errors on the primary model
|
||||||
|
/// automatically fall through to the fallback.
|
||||||
|
pub fallback_model: Option<String>,
|
||||||
|
/// Maximum number of retries for transient errors (default: 3).
|
||||||
|
/// With the default of 3, the provider makes up to 4 total attempts
|
||||||
|
/// (1 initial + 3 retries) before giving up.
|
||||||
|
pub max_retries: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LlmConfig {
|
impl LlmConfig {
|
||||||
@@ -356,6 +442,8 @@ impl LlmConfig {
|
|||||||
.unwrap_or_else(default_session_path),
|
.unwrap_or_else(default_session_path),
|
||||||
api_mode,
|
api_mode,
|
||||||
api_key: nearai_api_key,
|
api_key: nearai_api_key,
|
||||||
|
fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?,
|
||||||
|
max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Resolve provider-specific configs based on backend
|
// Resolve provider-specific configs based on backend
|
||||||
@@ -768,51 +856,35 @@ impl std::fmt::Debug for SecretsConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SecretsConfig {
|
impl SecretsConfig {
|
||||||
async fn resolve(bootstrap: &crate::bootstrap::BootstrapConfig) -> Result<Self, ConfigError> {
|
/// Auto-detect secrets master key from env var, then OS keychain.
|
||||||
|
///
|
||||||
|
/// Sequential probe: SECRETS_MASTER_KEY env var first, then OS keychain.
|
||||||
|
/// No saved "source" needed; just try each source in order.
|
||||||
|
async fn resolve() -> Result<Self, ConfigError> {
|
||||||
use crate::settings::KeySource;
|
use crate::settings::KeySource;
|
||||||
|
|
||||||
let (master_key, source) = if let Some(env_key) = optional_env("SECRETS_MASTER_KEY")? {
|
let (master_key, source) = if let Some(env_key) = optional_env("SECRETS_MASTER_KEY")? {
|
||||||
(Some(SecretString::from(env_key)), KeySource::Env)
|
(Some(SecretString::from(env_key)), KeySource::Env)
|
||||||
} else {
|
} else {
|
||||||
match bootstrap.secrets_master_key_source {
|
// Probe the OS keychain; if a key is stored, use it
|
||||||
KeySource::Keychain => {
|
match crate::secrets::keychain::get_master_key().await {
|
||||||
// Try to load from OS keychain (async on Linux)
|
Ok(key_bytes) => {
|
||||||
match crate::secrets::keychain::get_master_key().await {
|
let key_hex: String = key_bytes.iter().map(|b| format!("{:02x}", b)).collect();
|
||||||
Ok(key_bytes) => {
|
(Some(SecretString::from(key_hex)), KeySource::Keychain)
|
||||||
let key_hex: String =
|
|
||||||
key_bytes.iter().map(|b| format!("{:02x}", b)).collect();
|
|
||||||
(Some(SecretString::from(key_hex)), KeySource::Keychain)
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
// Keychain configured but key not found
|
|
||||||
// This might happen if keychain was cleared
|
|
||||||
tracing::warn!(
|
|
||||||
"Secrets configured for keychain but key not found. \
|
|
||||||
Run 'ironclaw onboard' to reconfigure."
|
|
||||||
);
|
|
||||||
(None, KeySource::None)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
KeySource::Env => {
|
Err(_) => (None, KeySource::None),
|
||||||
tracing::warn!(
|
|
||||||
"Secrets configured for env var but SECRETS_MASTER_KEY not set."
|
|
||||||
);
|
|
||||||
(None, KeySource::None)
|
|
||||||
}
|
|
||||||
KeySource::None => (None, KeySource::None),
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
let enabled = master_key.is_some();
|
let enabled = master_key.is_some();
|
||||||
|
|
||||||
if let Some(ref key) = master_key {
|
if let Some(ref key) = master_key
|
||||||
if key.expose_secret().len() < 32 {
|
&& key.expose_secret().len() < 32
|
||||||
return Err(ConfigError::InvalidValue {
|
{
|
||||||
key: "SECRETS_MASTER_KEY".to_string(),
|
return Err(ConfigError::InvalidValue {
|
||||||
message: "must be at least 32 bytes for AES-256-GCM".to_string(),
|
key: "SECRETS_MASTER_KEY".to_string(),
|
||||||
});
|
message: "must be at least 32 bytes for AES-256-GCM".to_string(),
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,549 @@
|
|||||||
|
//! SQLite-dialect migrations for the libSQL/Turso backend.
|
||||||
|
//!
|
||||||
|
//! Consolidates all PostgreSQL migrations (V1-V8) into a single SQLite-compatible
|
||||||
|
//! schema. Run once on database creation; idempotent via `IF NOT EXISTS`.
|
||||||
|
|
||||||
|
/// Consolidated schema for libSQL.
|
||||||
|
///
|
||||||
|
/// Translates PostgreSQL types and features:
|
||||||
|
/// - `UUID` -> `TEXT` (store as hex string)
|
||||||
|
/// - `TIMESTAMPTZ` -> `TEXT` (ISO-8601)
|
||||||
|
/// - `JSONB` -> `TEXT` (JSON encoded)
|
||||||
|
/// - `BYTEA` -> `BLOB`
|
||||||
|
/// - `NUMERIC` -> `TEXT` (preserve precision for rust_decimal)
|
||||||
|
/// - `TEXT[]` -> `TEXT` (JSON array)
|
||||||
|
/// - `VECTOR(1536)` -> `F32_BLOB(1536)` (libsql native)
|
||||||
|
/// - `TSVECTOR` -> FTS5 virtual table
|
||||||
|
/// - `BIGSERIAL` -> `INTEGER PRIMARY KEY AUTOINCREMENT`
|
||||||
|
/// - PL/pgSQL functions -> SQLite triggers
|
||||||
|
pub const SCHEMA: &str = r#"
|
||||||
|
|
||||||
|
-- ==================== Migration tracking ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS _migrations (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ==================== Conversations ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS conversations (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
channel TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
thread_id TEXT,
|
||||||
|
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
last_activity TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
metadata TEXT NOT NULL DEFAULT '{}'
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conversations_channel ON conversations(channel);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conversations_last_activity ON conversations(last_activity);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS conversation_messages (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||||
|
role TEXT NOT NULL,
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_conversation_messages_conversation
|
||||||
|
ON conversation_messages(conversation_id);
|
||||||
|
|
||||||
|
-- ==================== Agent Jobs ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS agent_jobs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
marketplace_job_id TEXT,
|
||||||
|
conversation_id TEXT REFERENCES conversations(id),
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
category TEXT,
|
||||||
|
status TEXT NOT NULL,
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
user_id TEXT NOT NULL DEFAULT 'default',
|
||||||
|
project_dir TEXT,
|
||||||
|
job_mode TEXT NOT NULL DEFAULT 'worker',
|
||||||
|
budget_amount TEXT,
|
||||||
|
budget_token TEXT,
|
||||||
|
bid_amount TEXT,
|
||||||
|
estimated_cost TEXT,
|
||||||
|
estimated_time_secs INTEGER,
|
||||||
|
estimated_value TEXT,
|
||||||
|
actual_cost TEXT,
|
||||||
|
actual_time_secs INTEGER,
|
||||||
|
success INTEGER,
|
||||||
|
failure_reason TEXT,
|
||||||
|
stuck_since TEXT,
|
||||||
|
repair_attempts INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
started_at TEXT,
|
||||||
|
completed_at TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_status ON agent_jobs(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_marketplace ON agent_jobs(marketplace_job_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_conversation ON agent_jobs(conversation_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_source ON agent_jobs(source);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_user ON agent_jobs(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_created ON agent_jobs(created_at DESC);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS job_actions (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||||
|
sequence_num INTEGER NOT NULL,
|
||||||
|
tool_name TEXT NOT NULL,
|
||||||
|
input TEXT NOT NULL,
|
||||||
|
output_raw TEXT,
|
||||||
|
output_sanitized TEXT,
|
||||||
|
sanitization_warnings TEXT,
|
||||||
|
cost TEXT,
|
||||||
|
duration_ms INTEGER,
|
||||||
|
success INTEGER NOT NULL,
|
||||||
|
error_message TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE(job_id, sequence_num)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_job_actions_job_id ON job_actions(job_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_job_actions_tool ON job_actions(tool_name);
|
||||||
|
|
||||||
|
-- ==================== Dynamic Tools ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS dynamic_tools (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL UNIQUE,
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
parameters_schema TEXT NOT NULL,
|
||||||
|
code TEXT NOT NULL,
|
||||||
|
sandbox_config TEXT NOT NULL,
|
||||||
|
created_by_job_id TEXT REFERENCES agent_jobs(id),
|
||||||
|
success_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
failure_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_error TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_dynamic_tools_status ON dynamic_tools(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_dynamic_tools_name ON dynamic_tools(name);
|
||||||
|
|
||||||
|
-- ==================== LLM Calls ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS llm_calls (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
job_id TEXT REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||||
|
conversation_id TEXT REFERENCES conversations(id),
|
||||||
|
provider TEXT NOT NULL,
|
||||||
|
model TEXT NOT NULL,
|
||||||
|
input_tokens INTEGER NOT NULL,
|
||||||
|
output_tokens INTEGER NOT NULL,
|
||||||
|
cost TEXT NOT NULL,
|
||||||
|
purpose TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_llm_calls_job ON llm_calls(job_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_llm_calls_conversation ON llm_calls(conversation_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_llm_calls_provider ON llm_calls(provider);
|
||||||
|
|
||||||
|
-- ==================== Estimation ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS estimation_snapshots (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||||
|
category TEXT NOT NULL,
|
||||||
|
tool_names TEXT NOT NULL DEFAULT '[]',
|
||||||
|
estimated_cost TEXT NOT NULL,
|
||||||
|
actual_cost TEXT,
|
||||||
|
estimated_time_secs INTEGER NOT NULL,
|
||||||
|
actual_time_secs INTEGER,
|
||||||
|
estimated_value TEXT NOT NULL,
|
||||||
|
actual_value TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_estimation_category ON estimation_snapshots(category);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_estimation_job ON estimation_snapshots(job_id);
|
||||||
|
|
||||||
|
-- ==================== Self Repair ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS repair_attempts (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
target_type TEXT NOT NULL,
|
||||||
|
target_id TEXT NOT NULL,
|
||||||
|
diagnosis TEXT NOT NULL,
|
||||||
|
action_taken TEXT NOT NULL,
|
||||||
|
success INTEGER NOT NULL,
|
||||||
|
error_message TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_repair_attempts_target ON repair_attempts(target_type, target_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_repair_attempts_created ON repair_attempts(created_at);
|
||||||
|
|
||||||
|
-- ==================== Workspace: Memory Documents ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS memory_documents (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
agent_id TEXT,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
metadata TEXT NOT NULL DEFAULT '{}',
|
||||||
|
UNIQUE (user_id, agent_id, path)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_documents_user ON memory_documents(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_documents_path ON memory_documents(user_id, path);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_documents_updated ON memory_documents(updated_at DESC);
|
||||||
|
|
||||||
|
-- Trigger to auto-update updated_at on memory_documents
|
||||||
|
CREATE TRIGGER IF NOT EXISTS update_memory_documents_updated_at
|
||||||
|
AFTER UPDATE ON memory_documents
|
||||||
|
FOR EACH ROW
|
||||||
|
WHEN NEW.updated_at = OLD.updated_at
|
||||||
|
BEGIN
|
||||||
|
UPDATE memory_documents SET updated_at = datetime('now') WHERE id = NEW.id;
|
||||||
|
END;
|
||||||
|
|
||||||
|
-- ==================== Workspace: Memory Chunks ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS memory_chunks (
|
||||||
|
_rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
id TEXT NOT NULL UNIQUE,
|
||||||
|
document_id TEXT NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE,
|
||||||
|
chunk_index INTEGER NOT NULL,
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
embedding F32_BLOB(1536),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE (document_id, chunk_index)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_chunks_document ON memory_chunks(document_id);
|
||||||
|
|
||||||
|
-- Vector index for semantic search (libSQL native)
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
|
||||||
|
ON memory_chunks (libsql_vector_idx(embedding));
|
||||||
|
|
||||||
|
-- FTS5 virtual table for full-text search
|
||||||
|
CREATE VIRTUAL TABLE IF NOT EXISTS memory_chunks_fts USING fts5(
|
||||||
|
content,
|
||||||
|
content='memory_chunks',
|
||||||
|
content_rowid='_rowid'
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Triggers to keep FTS5 in sync with memory_chunks
|
||||||
|
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_insert AFTER INSERT ON memory_chunks BEGIN
|
||||||
|
INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_delete AFTER DELETE ON memory_chunks BEGIN
|
||||||
|
INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content)
|
||||||
|
VALUES ('delete', old._rowid, old.content);
|
||||||
|
END;
|
||||||
|
|
||||||
|
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_update AFTER UPDATE ON memory_chunks BEGIN
|
||||||
|
INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content)
|
||||||
|
VALUES ('delete', old._rowid, old.content);
|
||||||
|
INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content);
|
||||||
|
END;
|
||||||
|
|
||||||
|
-- ==================== Workspace: Heartbeat State ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS heartbeat_state (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
agent_id TEXT,
|
||||||
|
last_run TEXT,
|
||||||
|
next_run TEXT,
|
||||||
|
interval_seconds INTEGER NOT NULL DEFAULT 1800,
|
||||||
|
enabled INTEGER NOT NULL DEFAULT 1,
|
||||||
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||||
|
last_checks TEXT NOT NULL DEFAULT '{}',
|
||||||
|
UNIQUE (user_id, agent_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_heartbeat_user ON heartbeat_state(user_id);
|
||||||
|
|
||||||
|
-- ==================== Secrets ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS secrets (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
encrypted_value BLOB NOT NULL,
|
||||||
|
key_salt BLOB NOT NULL,
|
||||||
|
provider TEXT,
|
||||||
|
expires_at TEXT,
|
||||||
|
last_used_at TEXT,
|
||||||
|
usage_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE (user_id, name)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secrets_user ON secrets(user_id);
|
||||||
|
|
||||||
|
-- ==================== WASM Tools ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS wasm_tools (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
version TEXT NOT NULL DEFAULT '1.0.0',
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
wasm_binary BLOB NOT NULL,
|
||||||
|
binary_hash BLOB NOT NULL,
|
||||||
|
parameters_schema TEXT NOT NULL,
|
||||||
|
source_url TEXT,
|
||||||
|
trust_level TEXT NOT NULL DEFAULT 'user',
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE (user_id, name, version)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_wasm_tools_user ON wasm_tools(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_wasm_tools_name ON wasm_tools(user_id, name);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_wasm_tools_status ON wasm_tools(status);
|
||||||
|
|
||||||
|
-- ==================== Tool Capabilities ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tool_capabilities (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE,
|
||||||
|
http_allowlist TEXT NOT NULL DEFAULT '[]',
|
||||||
|
allowed_secrets TEXT NOT NULL DEFAULT '[]',
|
||||||
|
tool_aliases TEXT NOT NULL DEFAULT '{}',
|
||||||
|
requests_per_minute INTEGER NOT NULL DEFAULT 60,
|
||||||
|
requests_per_hour INTEGER NOT NULL DEFAULT 1000,
|
||||||
|
max_request_body_bytes INTEGER NOT NULL DEFAULT 1048576,
|
||||||
|
max_response_body_bytes INTEGER NOT NULL DEFAULT 10485760,
|
||||||
|
workspace_read_prefixes TEXT NOT NULL DEFAULT '[]',
|
||||||
|
http_timeout_secs INTEGER NOT NULL DEFAULT 30,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE (wasm_tool_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ==================== Leak Detection Patterns ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS leak_detection_patterns (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL UNIQUE,
|
||||||
|
pattern TEXT NOT NULL,
|
||||||
|
severity TEXT NOT NULL DEFAULT 'high',
|
||||||
|
action TEXT NOT NULL DEFAULT 'block',
|
||||||
|
enabled INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ==================== Rate Limit State ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tool_rate_limit_state (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
minute_window_start TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
minute_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
hour_window_start TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
hour_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
UNIQUE (wasm_tool_id, user_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ==================== Secret Usage Audit Log ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS secret_usage_log (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
secret_id TEXT NOT NULL REFERENCES secrets(id) ON DELETE CASCADE,
|
||||||
|
wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
target_host TEXT NOT NULL,
|
||||||
|
target_path TEXT,
|
||||||
|
success INTEGER NOT NULL,
|
||||||
|
error_message TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secret_usage_user ON secret_usage_log(user_id);
|
||||||
|
|
||||||
|
-- ==================== Leak Detection Events ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS leak_detection_events (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
pattern_id TEXT REFERENCES leak_detection_patterns(id) ON DELETE SET NULL,
|
||||||
|
wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL,
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
action_taken TEXT NOT NULL,
|
||||||
|
context_preview TEXT,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ==================== Tool Failures ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tool_failures (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
tool_name TEXT NOT NULL UNIQUE,
|
||||||
|
error_message TEXT,
|
||||||
|
error_count INTEGER DEFAULT 1,
|
||||||
|
first_failure TEXT DEFAULT (datetime('now')),
|
||||||
|
last_failure TEXT DEFAULT (datetime('now')),
|
||||||
|
last_build_result TEXT,
|
||||||
|
repaired_at TEXT,
|
||||||
|
repair_attempts INTEGER DEFAULT 0
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_failures_name ON tool_failures(tool_name);
|
||||||
|
|
||||||
|
-- ==================== Job Events ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS job_events (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
job_id TEXT NOT NULL REFERENCES agent_jobs(id),
|
||||||
|
event_type TEXT NOT NULL,
|
||||||
|
data TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_job_events_job ON job_events(job_id, id);
|
||||||
|
|
||||||
|
-- ==================== Routines ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS routines (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
description TEXT NOT NULL DEFAULT '',
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
enabled INTEGER NOT NULL DEFAULT 1,
|
||||||
|
trigger_type TEXT NOT NULL,
|
||||||
|
trigger_config TEXT NOT NULL,
|
||||||
|
action_type TEXT NOT NULL,
|
||||||
|
action_config TEXT NOT NULL,
|
||||||
|
cooldown_secs INTEGER NOT NULL DEFAULT 300,
|
||||||
|
max_concurrent INTEGER NOT NULL DEFAULT 1,
|
||||||
|
dedup_window_secs INTEGER,
|
||||||
|
notify_channel TEXT,
|
||||||
|
notify_user TEXT NOT NULL DEFAULT 'default',
|
||||||
|
notify_on_success INTEGER NOT NULL DEFAULT 0,
|
||||||
|
notify_on_failure INTEGER NOT NULL DEFAULT 1,
|
||||||
|
notify_on_attention INTEGER NOT NULL DEFAULT 1,
|
||||||
|
state TEXT NOT NULL DEFAULT '{}',
|
||||||
|
last_run_at TEXT,
|
||||||
|
next_fire_at TEXT,
|
||||||
|
run_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
UNIQUE (user_id, name)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_routines_user ON routines(user_id);
|
||||||
|
|
||||||
|
-- ==================== Routine Runs ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS routine_runs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
routine_id TEXT NOT NULL REFERENCES routines(id) ON DELETE CASCADE,
|
||||||
|
trigger_type TEXT NOT NULL,
|
||||||
|
trigger_detail TEXT,
|
||||||
|
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
completed_at TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'running',
|
||||||
|
result_summary TEXT,
|
||||||
|
tokens_used INTEGER,
|
||||||
|
job_id TEXT REFERENCES agent_jobs(id),
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_routine_runs_routine ON routine_runs(routine_id);
|
||||||
|
|
||||||
|
-- ==================== Settings ====================
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS settings (
|
||||||
|
user_id TEXT NOT NULL,
|
||||||
|
key TEXT NOT NULL,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
|
PRIMARY KEY (user_id, key)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_settings_user ON settings(user_id);
|
||||||
|
|
||||||
|
-- ==================== Missing indexes (parity with PostgreSQL) ====================
|
||||||
|
|
||||||
|
-- agent_jobs
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_agent_jobs_stuck ON agent_jobs(stuck_since);
|
||||||
|
|
||||||
|
-- secrets
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secrets_provider ON secrets(provider);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secrets_expires ON secrets(expires_at);
|
||||||
|
|
||||||
|
-- wasm_tools
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_wasm_tools_trust ON wasm_tools(trust_level);
|
||||||
|
|
||||||
|
-- tool_capabilities
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_capabilities_tool ON tool_capabilities(wasm_tool_id);
|
||||||
|
|
||||||
|
-- leak_detection_patterns
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leak_patterns_enabled ON leak_detection_patterns(enabled);
|
||||||
|
|
||||||
|
-- tool_rate_limit_state
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_rate_limit_tool ON tool_rate_limit_state(wasm_tool_id);
|
||||||
|
|
||||||
|
-- secret_usage_log
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secret_usage_secret ON secret_usage_log(secret_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secret_usage_tool ON secret_usage_log(wasm_tool_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_secret_usage_created ON secret_usage_log(created_at DESC);
|
||||||
|
|
||||||
|
-- leak_detection_events
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leak_events_pattern ON leak_detection_events(pattern_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leak_events_tool ON leak_detection_events(wasm_tool_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leak_events_user ON leak_detection_events(user_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_leak_events_created ON leak_detection_events(created_at DESC);
|
||||||
|
|
||||||
|
-- tool_failures
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_failures_count ON tool_failures(error_count DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_tool_failures_unrepaired ON tool_failures(tool_name);
|
||||||
|
|
||||||
|
-- routines
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_routines_next_fire ON routines(next_fire_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_routines_event_triggers ON routines(user_id);
|
||||||
|
|
||||||
|
-- routine_runs
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_routine_runs_status ON routine_runs(status);
|
||||||
|
|
||||||
|
-- heartbeat_state
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_heartbeat_next_run ON heartbeat_state(next_run);
|
||||||
|
|
||||||
|
-- ==================== Seed data ====================
|
||||||
|
|
||||||
|
-- Pre-populate leak detection patterns (matches PostgreSQL V2 migration).
|
||||||
|
INSERT OR IGNORE INTO leak_detection_patterns (id, name, pattern, severity, action, enabled, created_at) VALUES
|
||||||
|
('550e8400-e29b-41d4-a716-446655440001', 'openai_api_key', 'sk-(?:proj-)?[a-zA-Z0-9]{20,}(?:T3BlbkFJ[a-zA-Z0-9_-]*)?', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440002', 'anthropic_api_key', 'sk-ant-api[a-zA-Z0-9_-]{90,}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440003', 'aws_access_key', 'AKIA[0-9A-Z]{16}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440004', 'aws_secret_key', '(?<![A-Za-z0-9/+=])[A-Za-z0-9/+=]{40}(?![A-Za-z0-9/+=])', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440005', 'github_token', 'gh[pousr]_[A-Za-z0-9_]{36,}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440006', 'github_fine_grained_pat', 'github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440007', 'stripe_api_key', 'sk_(?:live|test)_[a-zA-Z0-9]{24,}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440008', 'nearai_session', 'sess_[a-zA-Z0-9]{32,}', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440009', 'bearer_token', 'Bearer\s+[a-zA-Z0-9_-]{20,}', 'high', 'redact', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000a', 'pem_private_key', '-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000b', 'ssh_private_key', '-----BEGIN\s+(?:OPENSSH|EC|DSA)\s+PRIVATE\s+KEY-----', 'critical', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000c', 'google_api_key', 'AIza[0-9A-Za-z_-]{35}', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000d', 'slack_token', 'xox[baprs]-[0-9a-zA-Z-]{10,}', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000e', 'discord_token', '[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-44665544000f', 'twilio_api_key', 'SK[a-fA-F0-9]{32}', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440010', 'sendgrid_api_key', 'SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}', 'high', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440011', 'mailchimp_api_key', '[a-f0-9]{32}-us[0-9]{1,2}', 'medium', 'block', 1, datetime('now')),
|
||||||
|
('550e8400-e29b-41d4-a716-446655440012', 'high_entropy_hex', '(?<![a-fA-F0-9])[a-fA-F0-9]{64}(?![a-fA-F0-9])', 'medium', 'warn', 1, datetime('now'));
|
||||||
|
|
||||||
|
"#;
|
||||||
+538
@@ -0,0 +1,538 @@
|
|||||||
|
//! Database abstraction layer.
|
||||||
|
//!
|
||||||
|
//! Provides a backend-agnostic `Database` trait that unifies all persistence
|
||||||
|
//! operations. Two implementations exist behind feature flags:
|
||||||
|
//!
|
||||||
|
//! - `postgres` (default): Uses `deadpool-postgres` + `tokio-postgres`
|
||||||
|
//! - `libsql`: Uses libSQL (Turso's SQLite fork) for embedded/edge deployment
|
||||||
|
//!
|
||||||
|
//! The existing `Store`, `Repository`, `SecretsStore`, and `WasmToolStore`
|
||||||
|
//! types become thin wrappers that delegate to `Arc<dyn Database>`.
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
pub mod postgres;
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
pub mod libsql_backend;
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
pub mod libsql_migrations;
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use rust_decimal::Decimal;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::agent::BrokenTool;
|
||||||
|
use crate::agent::routine::{Routine, RoutineRun, RunStatus};
|
||||||
|
use crate::context::{ActionRecord, JobContext, JobState};
|
||||||
|
use crate::error::DatabaseError;
|
||||||
|
use crate::error::WorkspaceError;
|
||||||
|
use crate::history::{
|
||||||
|
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||||
|
SandboxJobSummary, SettingRow,
|
||||||
|
};
|
||||||
|
use crate::workspace::{MemoryChunk, MemoryDocument, WorkspaceEntry};
|
||||||
|
use crate::workspace::{SearchConfig, SearchResult};
|
||||||
|
|
||||||
|
/// Create a database backend from configuration, run migrations, and return it.
|
||||||
|
///
|
||||||
|
/// This is the shared helper for CLI commands and other call sites that need
|
||||||
|
/// a simple `Arc<dyn Database>` without retaining backend-specific handles
|
||||||
|
/// (e.g., `pg_pool` or `libsql_conn` for the secrets store). The main agent
|
||||||
|
/// startup in `main.rs` uses its own initialization block because it also
|
||||||
|
/// captures those backend-specific handles.
|
||||||
|
pub async fn connect_from_config(
|
||||||
|
config: &crate::config::DatabaseConfig,
|
||||||
|
) -> Result<Arc<dyn Database>, DatabaseError> {
|
||||||
|
match config.backend {
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
crate::config::DatabaseBackend::LibSql => {
|
||||||
|
use secrecy::ExposeSecret as _;
|
||||||
|
|
||||||
|
let default_path = crate::config::default_libsql_path();
|
||||||
|
let db_path = config.libsql_path.as_deref().unwrap_or(&default_path);
|
||||||
|
|
||||||
|
let backend = if let Some(ref url) = config.libsql_url {
|
||||||
|
let token = config.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||||
|
DatabaseError::Pool(
|
||||||
|
"LIBSQL_AUTH_TOKEN required when LIBSQL_URL is set".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
libsql_backend::LibSqlBackend::new_remote_replica(
|
||||||
|
db_path,
|
||||||
|
url,
|
||||||
|
token.expose_secret(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| DatabaseError::Pool(e.to_string()))?
|
||||||
|
} else {
|
||||||
|
libsql_backend::LibSqlBackend::new_local(db_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| DatabaseError::Pool(e.to_string()))?
|
||||||
|
};
|
||||||
|
backend.run_migrations().await?;
|
||||||
|
Ok(Arc::new(backend))
|
||||||
|
}
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
_ => {
|
||||||
|
let pg = postgres::PgBackend::new(config)
|
||||||
|
.await
|
||||||
|
.map_err(|e| DatabaseError::Pool(e.to_string()))?;
|
||||||
|
pg.run_migrations().await?;
|
||||||
|
Ok(Arc::new(pg))
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "postgres"))]
|
||||||
|
_ => Err(DatabaseError::Pool(
|
||||||
|
"No database backend available. Enable 'postgres' or 'libsql' feature.".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Backend-agnostic database trait.
|
||||||
|
///
|
||||||
|
/// Combines all persistence operations from Store, Repository, and related
|
||||||
|
/// stores into a single trait that can be implemented for different backends.
|
||||||
|
#[async_trait]
|
||||||
|
pub trait Database: Send + Sync {
|
||||||
|
/// Run schema migrations for this backend.
|
||||||
|
async fn run_migrations(&self) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Conversations ====================
|
||||||
|
|
||||||
|
/// Create a new conversation.
|
||||||
|
async fn create_conversation(
|
||||||
|
&self,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
thread_id: Option<&str>,
|
||||||
|
) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update conversation last activity.
|
||||||
|
async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Add a message to a conversation.
|
||||||
|
async fn add_conversation_message(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
role: &str,
|
||||||
|
content: &str,
|
||||||
|
) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
/// Ensure a conversation row exists (upsert).
|
||||||
|
async fn ensure_conversation(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
thread_id: Option<&str>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// List conversations with a title preview.
|
||||||
|
async fn list_conversations_with_preview(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
channel: &str,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<ConversationSummary>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get or create the singleton assistant conversation.
|
||||||
|
async fn get_or_create_assistant_conversation(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
channel: &str,
|
||||||
|
) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
/// Create a conversation with specific metadata.
|
||||||
|
async fn create_conversation_with_metadata(
|
||||||
|
&self,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
metadata: &serde_json::Value,
|
||||||
|
) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
/// Load messages with cursor-based pagination.
|
||||||
|
async fn list_conversation_messages_paginated(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
before: Option<DateTime<Utc>>,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<(Vec<ConversationMessage>, bool), DatabaseError>;
|
||||||
|
|
||||||
|
/// Merge a single key into conversation metadata.
|
||||||
|
async fn update_conversation_metadata_field(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Read conversation metadata.
|
||||||
|
async fn get_conversation_metadata(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
) -> Result<Option<serde_json::Value>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Load all messages for a conversation.
|
||||||
|
async fn list_conversation_messages(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
) -> Result<Vec<ConversationMessage>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Check if a conversation belongs to a specific user.
|
||||||
|
async fn conversation_belongs_to_user(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Jobs ====================
|
||||||
|
|
||||||
|
/// Save a job context.
|
||||||
|
async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get a job by ID.
|
||||||
|
async fn get_job(&self, id: Uuid) -> Result<Option<JobContext>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update job status.
|
||||||
|
async fn update_job_status(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: JobState,
|
||||||
|
failure_reason: Option<&str>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Mark job as stuck.
|
||||||
|
async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get stuck jobs.
|
||||||
|
async fn get_stuck_jobs(&self) -> Result<Vec<Uuid>, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Actions ====================
|
||||||
|
|
||||||
|
/// Save a job action.
|
||||||
|
async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get actions for a job.
|
||||||
|
async fn get_job_actions(&self, job_id: Uuid) -> Result<Vec<ActionRecord>, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== LLM Calls ====================
|
||||||
|
|
||||||
|
/// Record an LLM call.
|
||||||
|
async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Estimation Snapshots ====================
|
||||||
|
|
||||||
|
/// Save an estimation snapshot.
|
||||||
|
async fn save_estimation_snapshot(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
category: &str,
|
||||||
|
tool_names: &[String],
|
||||||
|
estimated_cost: Decimal,
|
||||||
|
estimated_time_secs: i32,
|
||||||
|
estimated_value: Decimal,
|
||||||
|
) -> Result<Uuid, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update estimation snapshot with actual values.
|
||||||
|
async fn update_estimation_actuals(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
actual_cost: Decimal,
|
||||||
|
actual_time_secs: i32,
|
||||||
|
actual_value: Option<Decimal>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Sandbox Jobs ====================
|
||||||
|
|
||||||
|
/// Insert a new sandbox job.
|
||||||
|
async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get a sandbox job by ID.
|
||||||
|
async fn get_sandbox_job(&self, id: Uuid) -> Result<Option<SandboxJobRecord>, DatabaseError>;
|
||||||
|
|
||||||
|
/// List all sandbox jobs, most recent first.
|
||||||
|
async fn list_sandbox_jobs(&self) -> Result<Vec<SandboxJobRecord>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update sandbox job status.
|
||||||
|
async fn update_sandbox_job_status(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
success: Option<bool>,
|
||||||
|
message: Option<&str>,
|
||||||
|
started_at: Option<DateTime<Utc>>,
|
||||||
|
completed_at: Option<DateTime<Utc>>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Mark stale sandbox jobs as interrupted.
|
||||||
|
async fn cleanup_stale_sandbox_jobs(&self) -> Result<u64, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get sandbox job summary.
|
||||||
|
async fn sandbox_job_summary(&self) -> Result<SandboxJobSummary, DatabaseError>;
|
||||||
|
|
||||||
|
/// List sandbox jobs for a specific user, most recent first.
|
||||||
|
async fn list_sandbox_jobs_for_user(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<Vec<SandboxJobRecord>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get sandbox job summary for a specific user.
|
||||||
|
async fn sandbox_job_summary_for_user(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<SandboxJobSummary, DatabaseError>;
|
||||||
|
|
||||||
|
/// Check if a sandbox job belongs to a specific user.
|
||||||
|
async fn sandbox_job_belongs_to_user(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update sandbox job mode.
|
||||||
|
async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get sandbox job mode.
|
||||||
|
async fn get_sandbox_job_mode(&self, id: Uuid) -> Result<Option<String>, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Job Events ====================
|
||||||
|
|
||||||
|
/// Persist a job event.
|
||||||
|
async fn save_job_event(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
event_type: &str,
|
||||||
|
data: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Load all job events.
|
||||||
|
async fn list_job_events(&self, job_id: Uuid) -> Result<Vec<JobEventRecord>, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Routines ====================
|
||||||
|
|
||||||
|
/// Create a new routine.
|
||||||
|
async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get a routine by ID.
|
||||||
|
async fn get_routine(&self, id: Uuid) -> Result<Option<Routine>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get a routine by user_id and name.
|
||||||
|
async fn get_routine_by_name(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<Option<Routine>, DatabaseError>;
|
||||||
|
|
||||||
|
/// List routines for a user.
|
||||||
|
async fn list_routines(&self, user_id: &str) -> Result<Vec<Routine>, DatabaseError>;
|
||||||
|
|
||||||
|
/// List all enabled event routines.
|
||||||
|
async fn list_event_routines(&self) -> Result<Vec<Routine>, DatabaseError>;
|
||||||
|
|
||||||
|
/// List due cron routines.
|
||||||
|
async fn list_due_cron_routines(&self) -> Result<Vec<Routine>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Update a routine.
|
||||||
|
async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Update runtime state after a routine fires.
|
||||||
|
async fn update_routine_runtime(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
last_run_at: DateTime<Utc>,
|
||||||
|
next_fire_at: Option<DateTime<Utc>>,
|
||||||
|
run_count: u64,
|
||||||
|
consecutive_failures: u32,
|
||||||
|
state: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Delete a routine.
|
||||||
|
async fn delete_routine(&self, id: Uuid) -> Result<bool, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Routine Runs ====================
|
||||||
|
|
||||||
|
/// Record a routine run starting.
|
||||||
|
async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Complete a routine run.
|
||||||
|
async fn complete_routine_run(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: RunStatus,
|
||||||
|
result_summary: Option<&str>,
|
||||||
|
tokens_used: Option<i32>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// List recent runs for a routine.
|
||||||
|
async fn list_routine_runs(
|
||||||
|
&self,
|
||||||
|
routine_id: Uuid,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<RoutineRun>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Count currently running runs for a routine.
|
||||||
|
async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result<i64, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Tool Failures ====================
|
||||||
|
|
||||||
|
/// Record a tool failure (upsert).
|
||||||
|
async fn record_tool_failure(
|
||||||
|
&self,
|
||||||
|
tool_name: &str,
|
||||||
|
error_message: &str,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Get broken tools exceeding threshold.
|
||||||
|
async fn get_broken_tools(&self, threshold: i32) -> Result<Vec<BrokenTool>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Mark a tool as repaired.
|
||||||
|
async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Increment repair attempts.
|
||||||
|
async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Settings ====================
|
||||||
|
|
||||||
|
/// Get a single setting.
|
||||||
|
async fn get_setting(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> Result<Option<serde_json::Value>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get a single setting with metadata.
|
||||||
|
async fn get_setting_full(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> Result<Option<SettingRow>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Set a single setting (upsert).
|
||||||
|
async fn set_setting(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Delete a single setting.
|
||||||
|
async fn delete_setting(&self, user_id: &str, key: &str) -> Result<bool, DatabaseError>;
|
||||||
|
|
||||||
|
/// List all settings for a user.
|
||||||
|
async fn list_settings(&self, user_id: &str) -> Result<Vec<SettingRow>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Get all settings as a flat map.
|
||||||
|
async fn get_all_settings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<HashMap<String, serde_json::Value>, DatabaseError>;
|
||||||
|
|
||||||
|
/// Bulk-write settings atomically.
|
||||||
|
async fn set_all_settings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
settings: &HashMap<String, serde_json::Value>,
|
||||||
|
) -> Result<(), DatabaseError>;
|
||||||
|
|
||||||
|
/// Check if settings exist for a user.
|
||||||
|
async fn has_settings(&self, user_id: &str) -> Result<bool, DatabaseError>;
|
||||||
|
|
||||||
|
// ==================== Workspace: Documents ====================
|
||||||
|
|
||||||
|
/// Get a document by path.
|
||||||
|
async fn get_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<MemoryDocument, WorkspaceError>;
|
||||||
|
|
||||||
|
/// Get a document by ID.
|
||||||
|
async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError>;
|
||||||
|
|
||||||
|
/// Get or create a document by path.
|
||||||
|
async fn get_or_create_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<MemoryDocument, WorkspaceError>;
|
||||||
|
|
||||||
|
/// Update a document's content.
|
||||||
|
async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError>;
|
||||||
|
|
||||||
|
/// Delete a document by path.
|
||||||
|
async fn delete_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<(), WorkspaceError>;
|
||||||
|
|
||||||
|
/// List files and directories in a directory path.
|
||||||
|
async fn list_directory(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
directory: &str,
|
||||||
|
) -> Result<Vec<WorkspaceEntry>, WorkspaceError>;
|
||||||
|
|
||||||
|
/// List all file paths in the workspace.
|
||||||
|
async fn list_all_paths(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
) -> Result<Vec<String>, WorkspaceError>;
|
||||||
|
|
||||||
|
/// List all documents for a user.
|
||||||
|
async fn list_documents(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
) -> Result<Vec<MemoryDocument>, WorkspaceError>;
|
||||||
|
|
||||||
|
// ==================== Workspace: Chunks ====================
|
||||||
|
|
||||||
|
/// Delete all chunks for a document.
|
||||||
|
async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError>;
|
||||||
|
|
||||||
|
/// Insert a chunk.
|
||||||
|
async fn insert_chunk(
|
||||||
|
&self,
|
||||||
|
document_id: Uuid,
|
||||||
|
chunk_index: i32,
|
||||||
|
content: &str,
|
||||||
|
embedding: Option<&[f32]>,
|
||||||
|
) -> Result<Uuid, WorkspaceError>;
|
||||||
|
|
||||||
|
/// Update a chunk's embedding.
|
||||||
|
async fn update_chunk_embedding(
|
||||||
|
&self,
|
||||||
|
chunk_id: Uuid,
|
||||||
|
embedding: &[f32],
|
||||||
|
) -> Result<(), WorkspaceError>;
|
||||||
|
|
||||||
|
/// Get chunks without embeddings for backfilling.
|
||||||
|
async fn get_chunks_without_embeddings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<MemoryChunk>, WorkspaceError>;
|
||||||
|
|
||||||
|
// ==================== Workspace: Search ====================
|
||||||
|
|
||||||
|
/// Perform hybrid search combining FTS and vector similarity.
|
||||||
|
async fn hybrid_search(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
query: &str,
|
||||||
|
embedding: Option<&[f32]>,
|
||||||
|
config: &SearchConfig,
|
||||||
|
) -> Result<Vec<SearchResult>, WorkspaceError>;
|
||||||
|
}
|
||||||
@@ -0,0 +1,627 @@
|
|||||||
|
//! PostgreSQL backend for the Database trait.
|
||||||
|
//!
|
||||||
|
//! Delegates to the existing `Store` (history) and `Repository` (workspace)
|
||||||
|
//! implementations, avoiding SQL duplication.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use deadpool_postgres::Pool;
|
||||||
|
use rust_decimal::Decimal;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::agent::BrokenTool;
|
||||||
|
use crate::agent::routine::{Routine, RoutineRun, RunStatus};
|
||||||
|
use crate::config::DatabaseConfig;
|
||||||
|
use crate::context::{ActionRecord, JobContext, JobState};
|
||||||
|
use crate::db::Database;
|
||||||
|
use crate::error::{DatabaseError, WorkspaceError};
|
||||||
|
use crate::history::{
|
||||||
|
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||||
|
SandboxJobSummary, SettingRow, Store,
|
||||||
|
};
|
||||||
|
use crate::workspace::{
|
||||||
|
MemoryChunk, MemoryDocument, Repository, SearchConfig, SearchResult, WorkspaceEntry,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// PostgreSQL database backend.
|
||||||
|
///
|
||||||
|
/// Wraps the existing `Store` (for history/conversations/jobs/routines/settings)
|
||||||
|
/// and `Repository` (for workspace documents/chunks/search) to implement the
|
||||||
|
/// unified `Database` trait.
|
||||||
|
pub struct PgBackend {
|
||||||
|
store: Store,
|
||||||
|
repo: Repository,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PgBackend {
|
||||||
|
/// Create a new PostgreSQL backend from configuration.
|
||||||
|
pub async fn new(config: &DatabaseConfig) -> Result<Self, DatabaseError> {
|
||||||
|
let store = Store::new(config).await?;
|
||||||
|
let repo = Repository::new(store.pool());
|
||||||
|
Ok(Self { store, repo })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get a clone of the connection pool.
|
||||||
|
///
|
||||||
|
/// Useful for sharing with components that still need raw pool access.
|
||||||
|
pub fn pool(&self) -> Pool {
|
||||||
|
self.store.pool()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Database for PgBackend {
|
||||||
|
async fn run_migrations(&self) -> Result<(), DatabaseError> {
|
||||||
|
self.store.run_migrations().await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Conversations ====================
|
||||||
|
|
||||||
|
async fn create_conversation(
|
||||||
|
&self,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
thread_id: Option<&str>,
|
||||||
|
) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.create_conversation(channel, user_id, thread_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError> {
|
||||||
|
self.store.touch_conversation(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn add_conversation_message(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
role: &str,
|
||||||
|
content: &str,
|
||||||
|
) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.add_conversation_message(conversation_id, role, content)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn ensure_conversation(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
thread_id: Option<&str>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.ensure_conversation(id, channel, user_id, thread_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_conversations_with_preview(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
channel: &str,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<ConversationSummary>, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.list_conversations_with_preview(user_id, channel, limit)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_or_create_assistant_conversation(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
channel: &str,
|
||||||
|
) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.get_or_create_assistant_conversation(user_id, channel)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn create_conversation_with_metadata(
|
||||||
|
&self,
|
||||||
|
channel: &str,
|
||||||
|
user_id: &str,
|
||||||
|
metadata: &serde_json::Value,
|
||||||
|
) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.create_conversation_with_metadata(channel, user_id, metadata)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_conversation_messages_paginated(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
before: Option<DateTime<Utc>>,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<(Vec<ConversationMessage>, bool), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.list_conversation_messages_paginated(conversation_id, before, limit)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_conversation_metadata_field(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.update_conversation_metadata_field(id, key, value)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_conversation_metadata(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
) -> Result<Option<serde_json::Value>, DatabaseError> {
|
||||||
|
self.store.get_conversation_metadata(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_conversation_messages(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
) -> Result<Vec<ConversationMessage>, DatabaseError> {
|
||||||
|
self.store.list_conversation_messages(conversation_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn conversation_belongs_to_user(
|
||||||
|
&self,
|
||||||
|
conversation_id: Uuid,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.conversation_belongs_to_user(conversation_id, user_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Jobs ====================
|
||||||
|
|
||||||
|
async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError> {
|
||||||
|
self.store.save_job(ctx).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_job(&self, id: Uuid) -> Result<Option<JobContext>, DatabaseError> {
|
||||||
|
self.store.get_job(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_job_status(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: JobState,
|
||||||
|
failure_reason: Option<&str>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.update_job_status(id, status, failure_reason)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError> {
|
||||||
|
self.store.mark_job_stuck(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_stuck_jobs(&self) -> Result<Vec<Uuid>, DatabaseError> {
|
||||||
|
self.store.get_stuck_jobs().await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Actions ====================
|
||||||
|
|
||||||
|
async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError> {
|
||||||
|
self.store.save_action(job_id, action).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_job_actions(&self, job_id: Uuid) -> Result<Vec<ActionRecord>, DatabaseError> {
|
||||||
|
self.store.get_job_actions(job_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== LLM Calls ====================
|
||||||
|
|
||||||
|
async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store.record_llm_call(record).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Estimation Snapshots ====================
|
||||||
|
|
||||||
|
async fn save_estimation_snapshot(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
category: &str,
|
||||||
|
tool_names: &[String],
|
||||||
|
estimated_cost: Decimal,
|
||||||
|
estimated_time_secs: i32,
|
||||||
|
estimated_value: Decimal,
|
||||||
|
) -> Result<Uuid, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.save_estimation_snapshot(
|
||||||
|
job_id,
|
||||||
|
category,
|
||||||
|
tool_names,
|
||||||
|
estimated_cost,
|
||||||
|
estimated_time_secs,
|
||||||
|
estimated_value,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_estimation_actuals(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
actual_cost: Decimal,
|
||||||
|
actual_time_secs: i32,
|
||||||
|
actual_value: Option<Decimal>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.update_estimation_actuals(id, actual_cost, actual_time_secs, actual_value)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Sandbox Jobs ====================
|
||||||
|
|
||||||
|
async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> {
|
||||||
|
self.store.save_sandbox_job(job).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_sandbox_job(&self, id: Uuid) -> Result<Option<SandboxJobRecord>, DatabaseError> {
|
||||||
|
self.store.get_sandbox_job(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_sandbox_jobs(&self) -> Result<Vec<SandboxJobRecord>, DatabaseError> {
|
||||||
|
self.store.list_sandbox_jobs().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_sandbox_job_status(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
success: Option<bool>,
|
||||||
|
message: Option<&str>,
|
||||||
|
started_at: Option<DateTime<Utc>>,
|
||||||
|
completed_at: Option<DateTime<Utc>>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.update_sandbox_job_status(id, status, success, message, started_at, completed_at)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_stale_sandbox_jobs(&self) -> Result<u64, DatabaseError> {
|
||||||
|
self.store.cleanup_stale_sandbox_jobs().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sandbox_job_summary(&self) -> Result<SandboxJobSummary, DatabaseError> {
|
||||||
|
self.store.sandbox_job_summary().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_sandbox_jobs_for_user(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<Vec<SandboxJobRecord>, DatabaseError> {
|
||||||
|
self.store.list_sandbox_jobs_for_user(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sandbox_job_summary_for_user(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<SandboxJobSummary, DatabaseError> {
|
||||||
|
self.store.sandbox_job_summary_for_user(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sandbox_job_belongs_to_user(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<bool, DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.sandbox_job_belongs_to_user(job_id, user_id)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError> {
|
||||||
|
self.store.update_sandbox_job_mode(id, mode).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_sandbox_job_mode(&self, id: Uuid) -> Result<Option<String>, DatabaseError> {
|
||||||
|
self.store.get_sandbox_job_mode(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Job Events ====================
|
||||||
|
|
||||||
|
async fn save_job_event(
|
||||||
|
&self,
|
||||||
|
job_id: Uuid,
|
||||||
|
event_type: &str,
|
||||||
|
data: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store.save_job_event(job_id, event_type, data).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_job_events(&self, job_id: Uuid) -> Result<Vec<JobEventRecord>, DatabaseError> {
|
||||||
|
self.store.list_job_events(job_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Routines ====================
|
||||||
|
|
||||||
|
async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||||
|
self.store.create_routine(routine).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_routine(&self, id: Uuid) -> Result<Option<Routine>, DatabaseError> {
|
||||||
|
self.store.get_routine(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_routine_by_name(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<Option<Routine>, DatabaseError> {
|
||||||
|
self.store.get_routine_by_name(user_id, name).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_routines(&self, user_id: &str) -> Result<Vec<Routine>, DatabaseError> {
|
||||||
|
self.store.list_routines(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_event_routines(&self) -> Result<Vec<Routine>, DatabaseError> {
|
||||||
|
self.store.list_event_routines().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_due_cron_routines(&self) -> Result<Vec<Routine>, DatabaseError> {
|
||||||
|
self.store.list_due_cron_routines().await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||||
|
self.store.update_routine(routine).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_routine_runtime(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
last_run_at: DateTime<Utc>,
|
||||||
|
next_fire_at: Option<DateTime<Utc>>,
|
||||||
|
run_count: u64,
|
||||||
|
consecutive_failures: u32,
|
||||||
|
state: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.update_routine_runtime(
|
||||||
|
id,
|
||||||
|
last_run_at,
|
||||||
|
next_fire_at,
|
||||||
|
run_count,
|
||||||
|
consecutive_failures,
|
||||||
|
state,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_routine(&self, id: Uuid) -> Result<bool, DatabaseError> {
|
||||||
|
self.store.delete_routine(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Routine Runs ====================
|
||||||
|
|
||||||
|
async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError> {
|
||||||
|
self.store.create_routine_run(run).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_routine_run(
|
||||||
|
&self,
|
||||||
|
id: Uuid,
|
||||||
|
status: RunStatus,
|
||||||
|
result_summary: Option<&str>,
|
||||||
|
tokens_used: Option<i32>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.complete_routine_run(id, status, result_summary, tokens_used)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_routine_runs(
|
||||||
|
&self,
|
||||||
|
routine_id: Uuid,
|
||||||
|
limit: i64,
|
||||||
|
) -> Result<Vec<RoutineRun>, DatabaseError> {
|
||||||
|
self.store.list_routine_runs(routine_id, limit).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result<i64, DatabaseError> {
|
||||||
|
self.store.count_running_routine_runs(routine_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Tool Failures ====================
|
||||||
|
|
||||||
|
async fn record_tool_failure(
|
||||||
|
&self,
|
||||||
|
tool_name: &str,
|
||||||
|
error_message: &str,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store
|
||||||
|
.record_tool_failure(tool_name, error_message)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_broken_tools(&self, threshold: i32) -> Result<Vec<BrokenTool>, DatabaseError> {
|
||||||
|
self.store.get_broken_tools(threshold).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError> {
|
||||||
|
self.store.mark_tool_repaired(tool_name).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError> {
|
||||||
|
self.store.increment_repair_attempts(tool_name).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Settings ====================
|
||||||
|
|
||||||
|
async fn get_setting(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> Result<Option<serde_json::Value>, DatabaseError> {
|
||||||
|
self.store.get_setting(user_id, key).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_setting_full(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
) -> Result<Option<SettingRow>, DatabaseError> {
|
||||||
|
self.store.get_setting_full(user_id, key).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_setting(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
key: &str,
|
||||||
|
value: &serde_json::Value,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store.set_setting(user_id, key, value).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_setting(&self, user_id: &str, key: &str) -> Result<bool, DatabaseError> {
|
||||||
|
self.store.delete_setting(user_id, key).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_settings(&self, user_id: &str) -> Result<Vec<SettingRow>, DatabaseError> {
|
||||||
|
self.store.list_settings(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_all_settings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
) -> Result<HashMap<String, serde_json::Value>, DatabaseError> {
|
||||||
|
self.store.get_all_settings(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn set_all_settings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
settings: &HashMap<String, serde_json::Value>,
|
||||||
|
) -> Result<(), DatabaseError> {
|
||||||
|
self.store.set_all_settings(user_id, settings).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn has_settings(&self, user_id: &str) -> Result<bool, DatabaseError> {
|
||||||
|
self.store.has_settings(user_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Workspace: Documents ====================
|
||||||
|
|
||||||
|
async fn get_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<MemoryDocument, WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.get_document_by_path(user_id, agent_id, path)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError> {
|
||||||
|
self.repo.get_document_by_id(id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_or_create_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<MemoryDocument, WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.get_or_create_document_by_path(user_id, agent_id, path)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> {
|
||||||
|
self.repo.update_document(id, content).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete_document_by_path(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
path: &str,
|
||||||
|
) -> Result<(), WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.delete_document_by_path(user_id, agent_id, path)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_directory(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
directory: &str,
|
||||||
|
) -> Result<Vec<WorkspaceEntry>, WorkspaceError> {
|
||||||
|
self.repo.list_directory(user_id, agent_id, directory).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_all_paths(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
) -> Result<Vec<String>, WorkspaceError> {
|
||||||
|
self.repo.list_all_paths(user_id, agent_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_documents(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
) -> Result<Vec<MemoryDocument>, WorkspaceError> {
|
||||||
|
self.repo.list_documents(user_id, agent_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Workspace: Chunks ====================
|
||||||
|
|
||||||
|
async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
|
||||||
|
self.repo.delete_chunks(document_id).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn insert_chunk(
|
||||||
|
&self,
|
||||||
|
document_id: Uuid,
|
||||||
|
chunk_index: i32,
|
||||||
|
content: &str,
|
||||||
|
embedding: Option<&[f32]>,
|
||||||
|
) -> Result<Uuid, WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.insert_chunk(document_id, chunk_index, content, embedding)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_chunk_embedding(
|
||||||
|
&self,
|
||||||
|
chunk_id: Uuid,
|
||||||
|
embedding: &[f32],
|
||||||
|
) -> Result<(), WorkspaceError> {
|
||||||
|
self.repo.update_chunk_embedding(chunk_id, embedding).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_chunks_without_embeddings(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<Vec<MemoryChunk>, WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.get_chunks_without_embeddings(user_id, agent_id, limit)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Workspace: Search ====================
|
||||||
|
|
||||||
|
async fn hybrid_search(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
agent_id: Option<Uuid>,
|
||||||
|
query: &str,
|
||||||
|
embedding: Option<&[f32]>,
|
||||||
|
config: &SearchConfig,
|
||||||
|
) -> Result<Vec<SearchResult>, WorkspaceError> {
|
||||||
|
self.repo
|
||||||
|
.hybrid_search(user_id, agent_id, query, embedding, config)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -87,14 +87,21 @@ pub enum DatabaseError {
|
|||||||
#[error("Serialization error: {0}")]
|
#[error("Serialization error: {0}")]
|
||||||
Serialization(String),
|
Serialization(String),
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
#[error("PostgreSQL error: {0}")]
|
#[error("PostgreSQL error: {0}")]
|
||||||
Postgres(#[from] tokio_postgres::Error),
|
Postgres(#[from] tokio_postgres::Error),
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
#[error("Pool build error: {0}")]
|
#[error("Pool build error: {0}")]
|
||||||
PoolBuild(#[from] deadpool_postgres::BuildError),
|
PoolBuild(#[from] deadpool_postgres::BuildError),
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
#[error("Pool runtime error: {0}")]
|
#[error("Pool runtime error: {0}")]
|
||||||
PoolRuntime(#[from] deadpool_postgres::PoolError),
|
PoolRuntime(#[from] deadpool_postgres::PoolError),
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
#[error("LibSQL error: {0}")]
|
||||||
|
LibSql(#[from] libsql::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Channel-related errors.
|
/// Channel-related errors.
|
||||||
|
|||||||
@@ -20,10 +20,6 @@ impl CostEstimator {
|
|||||||
|
|
||||||
// Default tool costs (in USD or equivalent)
|
// Default tool costs (in USD or equivalent)
|
||||||
tool_costs.insert("http".to_string(), dec!(0.0001)); // API call
|
tool_costs.insert("http".to_string(), dec!(0.0001)); // API call
|
||||||
tool_costs.insert("marketplace".to_string(), dec!(0.01)); // Gas costs
|
|
||||||
tool_costs.insert("ecommerce".to_string(), dec!(0.001)); // API call
|
|
||||||
tool_costs.insert("taskrabbit".to_string(), dec!(0.0)); // Cost comes from task itself
|
|
||||||
tool_costs.insert("restaurant".to_string(), dec!(0.001)); // API call
|
|
||||||
tool_costs.insert("echo".to_string(), dec!(0.0)); // Free
|
tool_costs.insert("echo".to_string(), dec!(0.0)); // Free
|
||||||
tool_costs.insert("time".to_string(), dec!(0.0)); // Free
|
tool_costs.insert("time".to_string(), dec!(0.0)); // Free
|
||||||
tool_costs.insert("json".to_string(), dec!(0.0)); // Free
|
tool_costs.insert("json".to_string(), dec!(0.0)); // Free
|
||||||
@@ -74,7 +70,7 @@ mod tests {
|
|||||||
let estimator = CostEstimator::new();
|
let estimator = CostEstimator::new();
|
||||||
|
|
||||||
assert_eq!(estimator.estimate_tool("echo"), dec!(0.0));
|
assert_eq!(estimator.estimate_tool("echo"), dec!(0.0));
|
||||||
assert_eq!(estimator.estimate_tool("marketplace"), dec!(0.01));
|
assert_eq!(estimator.estimate_tool("http"), dec!(0.0001));
|
||||||
assert!(estimator.estimate_tool("unknown") > dec!(0.0));
|
assert!(estimator.estimate_tool("unknown") > dec!(0.0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,10 +16,6 @@ impl TimeEstimator {
|
|||||||
|
|
||||||
// Default tool durations
|
// Default tool durations
|
||||||
tool_durations.insert("http".to_string(), Duration::from_secs(5));
|
tool_durations.insert("http".to_string(), Duration::from_secs(5));
|
||||||
tool_durations.insert("marketplace".to_string(), Duration::from_secs(10));
|
|
||||||
tool_durations.insert("ecommerce".to_string(), Duration::from_secs(8));
|
|
||||||
tool_durations.insert("taskrabbit".to_string(), Duration::from_secs(30)); // Just API, not task itself
|
|
||||||
tool_durations.insert("restaurant".to_string(), Duration::from_secs(5));
|
|
||||||
tool_durations.insert("echo".to_string(), Duration::from_millis(10));
|
tool_durations.insert("echo".to_string(), Duration::from_millis(10));
|
||||||
tool_durations.insert("time".to_string(), Duration::from_millis(1));
|
tool_durations.insert("time".to_string(), Duration::from_millis(1));
|
||||||
tool_durations.insert("json".to_string(), Duration::from_millis(5));
|
tool_durations.insert("json".to_string(), Duration::from_millis(5));
|
||||||
|
|||||||
@@ -144,12 +144,11 @@ impl SuccessEvaluator for RuleBasedEvaluator {
|
|||||||
|
|
||||||
// Check for critical errors
|
// Check for critical errors
|
||||||
for action in actions.iter().filter(|a| !a.success) {
|
for action in actions.iter().filter(|a| !a.success) {
|
||||||
if let Some(ref error) = action.error {
|
if let Some(ref error) = action.error
|
||||||
if error.to_lowercase().contains("critical")
|
&& (error.to_lowercase().contains("critical")
|
||||||
|| error.to_lowercase().contains("fatal")
|
|| error.to_lowercase().contains("fatal"))
|
||||||
{
|
{
|
||||||
issues.push(format!("Critical error in {}: {}", action.tool_name, error));
|
issues.push(format!("Critical error in {}: {}", action.tool_name, error));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+34
-32
@@ -57,7 +57,7 @@ pub struct ExtensionManager {
|
|||||||
_tunnel_url: Option<String>,
|
_tunnel_url: Option<String>,
|
||||||
user_id: String,
|
user_id: String,
|
||||||
/// Optional database store for DB-backed MCP config.
|
/// Optional database store for DB-backed MCP config.
|
||||||
store: Option<Arc<crate::history::Store>>,
|
store: Option<Arc<dyn crate::db::Database>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ExtensionManager {
|
impl ExtensionManager {
|
||||||
@@ -71,7 +71,7 @@ impl ExtensionManager {
|
|||||||
wasm_channels_dir: PathBuf,
|
wasm_channels_dir: PathBuf,
|
||||||
tunnel_url: Option<String>,
|
tunnel_url: Option<String>,
|
||||||
user_id: String,
|
user_id: String,
|
||||||
store: Option<Arc<crate::history::Store>>,
|
store: Option<Arc<dyn crate::db::Database>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
registry: ExtensionRegistry::new(),
|
registry: ExtensionRegistry::new(),
|
||||||
@@ -351,7 +351,7 @@ impl ExtensionManager {
|
|||||||
) -> Result<crate::tools::mcp::config::McpServersFile, crate::tools::mcp::config::ConfigError>
|
) -> Result<crate::tools::mcp::config::McpServersFile, crate::tools::mcp::config::ConfigError>
|
||||||
{
|
{
|
||||||
if let Some(ref store) = self.store {
|
if let Some(ref store) = self.store {
|
||||||
crate::tools::mcp::config::load_mcp_servers_from_db(store, &self.user_id).await
|
crate::tools::mcp::config::load_mcp_servers_from_db(store.as_ref(), &self.user_id).await
|
||||||
} else {
|
} else {
|
||||||
crate::tools::mcp::config::load_mcp_servers().await
|
crate::tools::mcp::config::load_mcp_servers().await
|
||||||
}
|
}
|
||||||
@@ -375,7 +375,8 @@ impl ExtensionManager {
|
|||||||
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
||||||
config.validate()?;
|
config.validate()?;
|
||||||
if let Some(ref store) = self.store {
|
if let Some(ref store) = self.store {
|
||||||
crate::tools::mcp::config::add_mcp_server_db(store, &self.user_id, config).await
|
crate::tools::mcp::config::add_mcp_server_db(store.as_ref(), &self.user_id, config)
|
||||||
|
.await
|
||||||
} else {
|
} else {
|
||||||
crate::tools::mcp::config::add_mcp_server(config).await
|
crate::tools::mcp::config::add_mcp_server(config).await
|
||||||
}
|
}
|
||||||
@@ -386,7 +387,8 @@ impl ExtensionManager {
|
|||||||
name: &str,
|
name: &str,
|
||||||
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
||||||
if let Some(ref store) = self.store {
|
if let Some(ref store) = self.store {
|
||||||
crate::tools::mcp::config::remove_mcp_server_db(store, &self.user_id, name).await
|
crate::tools::mcp::config::remove_mcp_server_db(store.as_ref(), &self.user_id, name)
|
||||||
|
.await
|
||||||
} else {
|
} else {
|
||||||
crate::tools::mcp::config::remove_mcp_server(name).await
|
crate::tools::mcp::config::remove_mcp_server(name).await
|
||||||
}
|
}
|
||||||
@@ -490,13 +492,13 @@ impl ExtensionManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check Content-Length header before downloading the full body
|
// Check Content-Length header before downloading the full body
|
||||||
if let Some(len) = response.content_length() {
|
if let Some(len) = response.content_length()
|
||||||
if len as usize > MAX_WASM_SIZE {
|
&& len as usize > MAX_WASM_SIZE
|
||||||
return Err(ExtensionError::InstallFailed(format!(
|
{
|
||||||
"WASM binary too large ({} bytes, max {} bytes)",
|
return Err(ExtensionError::InstallFailed(format!(
|
||||||
len, MAX_WASM_SIZE
|
"WASM binary too large ({} bytes, max {} bytes)",
|
||||||
)));
|
len, MAX_WASM_SIZE
|
||||||
}
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let bytes = response
|
let bytes = response
|
||||||
@@ -766,27 +768,27 @@ impl ExtensionManager {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Check env var first
|
// Check env var first
|
||||||
if let Some(ref env_var) = auth.env_var {
|
if let Some(ref env_var) = auth.env_var
|
||||||
if let Ok(value) = std::env::var(env_var) {
|
&& let Ok(value) = std::env::var(env_var)
|
||||||
// Store the env var value as a secret
|
{
|
||||||
let params = CreateSecretParams::new(&auth.secret_name, &value)
|
// Store the env var value as a secret
|
||||||
.with_provider(name.to_string());
|
let params =
|
||||||
self.secrets
|
CreateSecretParams::new(&auth.secret_name, &value).with_provider(name.to_string());
|
||||||
.create(&self.user_id, params)
|
self.secrets
|
||||||
.await
|
.create(&self.user_id, params)
|
||||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
.await
|
||||||
|
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||||
|
|
||||||
return Ok(AuthResult {
|
return Ok(AuthResult {
|
||||||
name: name.to_string(),
|
name: name.to_string(),
|
||||||
kind: ExtensionKind::WasmTool,
|
kind: ExtensionKind::WasmTool,
|
||||||
auth_url: None,
|
auth_url: None,
|
||||||
callback_type: None,
|
callback_type: None,
|
||||||
instructions: None,
|
instructions: None,
|
||||||
setup_url: None,
|
setup_url: None,
|
||||||
awaiting_token: false,
|
awaiting_token: false,
|
||||||
status: "authenticated".to_string(),
|
status: "authenticated".to_string(),
|
||||||
});
|
});
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if already authenticated
|
// Check if already authenticated
|
||||||
|
|||||||
+5
-1
@@ -5,11 +5,15 @@
|
|||||||
//! - Learning from past executions
|
//! - Learning from past executions
|
||||||
//! - Analytics and metrics
|
//! - Analytics and metrics
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
mod analytics;
|
mod analytics;
|
||||||
mod store;
|
mod store;
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub use analytics::{JobStats, ToolStats};
|
pub use analytics::{JobStats, ToolStats};
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
pub use store::Store;
|
||||||
pub use store::{
|
pub use store::{
|
||||||
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||||
SandboxJobSummary, Store,
|
SandboxJobSummary, SettingRow,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,13 +1,18 @@
|
|||||||
//! PostgreSQL store for persisting agent data.
|
//! PostgreSQL store for persisting agent data.
|
||||||
|
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use deadpool_postgres::{Config, Pool, Runtime};
|
use deadpool_postgres::{Config, Pool, Runtime};
|
||||||
use rust_decimal::Decimal;
|
use rust_decimal::Decimal;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use tokio_postgres::NoTls;
|
use tokio_postgres::NoTls;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use crate::config::DatabaseConfig;
|
use crate::config::DatabaseConfig;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use crate::context::{ActionRecord, JobContext, JobState};
|
use crate::context::{ActionRecord, JobContext, JobState};
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use crate::error::DatabaseError;
|
use crate::error::DatabaseError;
|
||||||
|
|
||||||
/// Record for an LLM call to be persisted.
|
/// Record for an LLM call to be persisted.
|
||||||
@@ -24,11 +29,18 @@ pub struct LlmCallRecord<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Database store for the agent.
|
/// Database store for the agent.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub struct Store {
|
pub struct Store {
|
||||||
pool: Pool,
|
pool: Pool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
|
/// Wrap an existing pool (useful when the caller already has a connection).
|
||||||
|
pub fn from_pool(pool: Pool) -> Self {
|
||||||
|
Self { pool }
|
||||||
|
}
|
||||||
|
|
||||||
/// Create a new store and connect to the database.
|
/// Create a new store and connect to the database.
|
||||||
pub async fn new(config: &DatabaseConfig) -> Result<Self, DatabaseError> {
|
pub async fn new(config: &DatabaseConfig) -> Result<Self, DatabaseError> {
|
||||||
let mut cfg = Config::new();
|
let mut cfg = Config::new();
|
||||||
@@ -144,7 +156,12 @@ impl Store {
|
|||||||
actual_cost, repair_attempts, created_at, started_at, completed_at
|
actual_cost, repair_attempts, created_at, started_at, completed_at
|
||||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
||||||
ON CONFLICT (id) DO UPDATE SET
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
title = EXCLUDED.title,
|
||||||
|
description = EXCLUDED.description,
|
||||||
|
category = EXCLUDED.category,
|
||||||
status = EXCLUDED.status,
|
status = EXCLUDED.status,
|
||||||
|
estimated_cost = EXCLUDED.estimated_cost,
|
||||||
|
estimated_time_secs = EXCLUDED.estimated_time_secs,
|
||||||
actual_cost = EXCLUDED.actual_cost,
|
actual_cost = EXCLUDED.actual_cost,
|
||||||
repair_attempts = EXCLUDED.repair_attempts,
|
repair_attempts = EXCLUDED.repair_attempts,
|
||||||
started_at = EXCLUDED.started_at,
|
started_at = EXCLUDED.started_at,
|
||||||
@@ -466,6 +483,7 @@ pub struct SandboxJobSummary {
|
|||||||
pub interrupted: usize,
|
pub interrupted: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Insert a new sandbox job into `agent_jobs`.
|
/// Insert a new sandbox job into `agent_jobs`.
|
||||||
pub async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> {
|
pub async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> {
|
||||||
@@ -742,6 +760,7 @@ pub struct JobEventRecord {
|
|||||||
pub created_at: DateTime<Utc>,
|
pub created_at: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Persist a job event (fire-and-forget from orchestrator handler).
|
/// Persist a job event (fire-and-forget from orchestrator handler).
|
||||||
pub async fn save_job_event(
|
pub async fn save_job_event(
|
||||||
@@ -814,10 +833,12 @@ impl Store {
|
|||||||
|
|
||||||
// ==================== Routines ====================
|
// ==================== Routines ====================
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use crate::agent::routine::{
|
use crate::agent::routine::{
|
||||||
NotifyConfig, Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger,
|
NotifyConfig, Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Create a new routine.
|
/// Create a new routine.
|
||||||
pub async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
pub async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||||
@@ -1118,6 +1139,7 @@ impl Store {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
||||||
let trigger_type: String = row.get("trigger_type");
|
let trigger_type: String = row.get("trigger_type");
|
||||||
let trigger_config: serde_json::Value = row.get("trigger_config");
|
let trigger_config: serde_json::Value = row.get("trigger_config");
|
||||||
@@ -1162,6 +1184,7 @@ fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn row_to_routine_run(row: &tokio_postgres::Row) -> Result<RoutineRun, DatabaseError> {
|
fn row_to_routine_run(row: &tokio_postgres::Row) -> Result<RoutineRun, DatabaseError> {
|
||||||
let status_str: String = row.get("status");
|
let status_str: String = row.get("status");
|
||||||
let status: RunStatus = status_str
|
let status: RunStatus = status_str
|
||||||
@@ -1207,6 +1230,7 @@ pub struct ConversationMessage {
|
|||||||
pub created_at: DateTime<Utc>,
|
pub created_at: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Ensure a conversation row exists for a given UUID.
|
/// Ensure a conversation row exists for a given UUID.
|
||||||
///
|
///
|
||||||
@@ -1477,6 +1501,7 @@ impl Store {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn parse_job_state(s: &str) -> JobState {
|
fn parse_job_state(s: &str) -> JobState {
|
||||||
match s {
|
match s {
|
||||||
"pending" => JobState::Pending,
|
"pending" => JobState::Pending,
|
||||||
@@ -1493,8 +1518,10 @@ fn parse_job_state(s: &str) -> JobState {
|
|||||||
|
|
||||||
// ==================== Tool Failures ====================
|
// ==================== Tool Failures ====================
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use crate::agent::BrokenTool;
|
use crate::agent::BrokenTool;
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Record a tool failure (upsert: increment count if exists).
|
/// Record a tool failure (upsert: increment count if exists).
|
||||||
pub async fn record_tool_failure(
|
pub async fn record_tool_failure(
|
||||||
@@ -1588,6 +1615,7 @@ pub struct SettingRow {
|
|||||||
pub updated_at: DateTime<Utc>,
|
pub updated_at: DateTime<Utc>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl Store {
|
impl Store {
|
||||||
/// Get a single setting by key.
|
/// Get a single setting by key.
|
||||||
pub async fn get_setting(
|
pub async fn get_setting(
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ pub mod channels;
|
|||||||
pub mod cli;
|
pub mod cli;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
pub mod context;
|
pub mod context;
|
||||||
|
pub mod db;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod estimation;
|
pub mod estimation;
|
||||||
pub mod evaluation;
|
pub mod evaluation;
|
||||||
@@ -58,6 +59,7 @@ pub mod secrets;
|
|||||||
pub mod settings;
|
pub mod settings;
|
||||||
pub mod setup;
|
pub mod setup;
|
||||||
pub mod tools;
|
pub mod tools;
|
||||||
|
pub mod tracing_fmt;
|
||||||
pub mod util;
|
pub mod util;
|
||||||
pub mod worker;
|
pub mod worker;
|
||||||
pub mod workspace;
|
pub mod workspace;
|
||||||
|
|||||||
@@ -0,0 +1,483 @@
|
|||||||
|
//! Multi-provider LLM failover.
|
||||||
|
//!
|
||||||
|
//! Wraps multiple LlmProvider instances and tries each in sequence
|
||||||
|
//! until one succeeds. Transparent to callers --- same LlmProvider trait.
|
||||||
|
|
||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use rust_decimal::Decimal;
|
||||||
|
|
||||||
|
use crate::error::LlmError;
|
||||||
|
use crate::llm::provider::{
|
||||||
|
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
|
||||||
|
ToolCompletionResponse,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Returns `true` if the error is transient and the request should be retried
|
||||||
|
/// on the next provider in the failover chain.
|
||||||
|
///
|
||||||
|
/// Retryable: `RequestFailed`, `RateLimited`, `InvalidResponse`,
|
||||||
|
/// `SessionRenewalFailed`, `ModelNotAvailable`, `Http`, `Io`.
|
||||||
|
///
|
||||||
|
/// `ModelNotAvailable` is retryable because the next provider in the chain may
|
||||||
|
/// offer a different model, so it's worth trying.
|
||||||
|
///
|
||||||
|
/// Non-retryable errors (`AuthFailed`, `SessionExpired`, `ContextLengthExceeded`)
|
||||||
|
/// propagate immediately because a different provider won't fix them.
|
||||||
|
fn is_retryable(err: &LlmError) -> bool {
|
||||||
|
matches!(
|
||||||
|
err,
|
||||||
|
LlmError::RequestFailed { .. }
|
||||||
|
| LlmError::RateLimited { .. }
|
||||||
|
| LlmError::InvalidResponse { .. }
|
||||||
|
| LlmError::SessionRenewalFailed { .. }
|
||||||
|
// ModelNotAvailable is retryable: the next provider may offer a different model.
|
||||||
|
| LlmError::ModelNotAvailable { .. }
|
||||||
|
| LlmError::Http(_)
|
||||||
|
| LlmError::Io(_)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An LLM provider that wraps multiple providers and tries each in sequence
|
||||||
|
/// on transient failures.
|
||||||
|
///
|
||||||
|
/// The first provider in the list is the primary. If it fails with a retryable
|
||||||
|
/// error, the next provider is tried, and so on. Non-retryable errors
|
||||||
|
/// (e.g. `AuthFailed`, `ContextLengthExceeded`) propagate immediately.
|
||||||
|
pub struct FailoverProvider {
|
||||||
|
providers: Vec<Arc<dyn LlmProvider>>,
|
||||||
|
/// Index of the provider that last handled a request successfully.
|
||||||
|
/// Used by `model_name()` and `cost_per_token()` so downstream cost
|
||||||
|
/// tracking reflects the provider that actually served the request.
|
||||||
|
last_used: AtomicUsize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FailoverProvider {
|
||||||
|
/// Create a new failover provider.
|
||||||
|
///
|
||||||
|
/// Returns an error if `providers` is empty.
|
||||||
|
pub fn new(providers: Vec<Arc<dyn LlmProvider>>) -> Result<Self, LlmError> {
|
||||||
|
if providers.is_empty() {
|
||||||
|
return Err(LlmError::RequestFailed {
|
||||||
|
provider: "failover".to_string(),
|
||||||
|
reason: "FailoverProvider requires at least one provider".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
providers,
|
||||||
|
last_used: AtomicUsize::new(0),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try each provider in sequence until one succeeds or all fail.
|
||||||
|
async fn try_providers<T, F, Fut>(&self, mut call: F) -> Result<T, LlmError>
|
||||||
|
where
|
||||||
|
F: FnMut(Arc<dyn LlmProvider>) -> Fut,
|
||||||
|
Fut: Future<Output = Result<T, LlmError>>,
|
||||||
|
{
|
||||||
|
let mut last_error: Option<LlmError> = None;
|
||||||
|
|
||||||
|
for (i, provider) in self.providers.iter().enumerate() {
|
||||||
|
let result = call(Arc::clone(provider)).await;
|
||||||
|
match result {
|
||||||
|
Ok(response) => {
|
||||||
|
self.last_used.store(i, Ordering::Relaxed);
|
||||||
|
return Ok(response);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
if !is_retryable(&err) {
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
if i + 1 < self.providers.len() {
|
||||||
|
tracing::warn!(
|
||||||
|
provider = %provider.model_name(),
|
||||||
|
error = %err,
|
||||||
|
next_provider = %self.providers[i + 1].model_name(),
|
||||||
|
"Provider failed with retryable error, trying next provider"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
last_error = Some(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: providers is non-empty (checked in `new`), so at least one
|
||||||
|
// iteration ran and `last_error` is `Some`.
|
||||||
|
Err(last_error.expect("providers list is non-empty"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmProvider for FailoverProvider {
|
||||||
|
fn model_name(&self) -> &str {
|
||||||
|
self.providers[self.last_used.load(Ordering::Relaxed)].model_name()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||||
|
self.providers[self.last_used.load(Ordering::Relaxed)].cost_per_token()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||||
|
self.try_providers(|provider| {
|
||||||
|
let req = request.clone();
|
||||||
|
async move { provider.complete(req).await }
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_with_tools(
|
||||||
|
&self,
|
||||||
|
request: ToolCompletionRequest,
|
||||||
|
) -> Result<ToolCompletionResponse, LlmError> {
|
||||||
|
self.try_providers(|provider| {
|
||||||
|
let req = request.clone();
|
||||||
|
async move { provider.complete_with_tools(req).await }
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||||
|
let mut all_models = Vec::new();
|
||||||
|
|
||||||
|
for provider in &self.providers {
|
||||||
|
match provider.list_models().await {
|
||||||
|
Ok(models) => all_models.extend(models),
|
||||||
|
Err(err) => {
|
||||||
|
tracing::warn!(
|
||||||
|
provider = %provider.model_name(),
|
||||||
|
error = %err,
|
||||||
|
"Failed to list models from provider, skipping"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
all_models.sort();
|
||||||
|
all_models.dedup();
|
||||||
|
Ok(all_models)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use crate::llm::provider::{CompletionResponse, FinishReason, ToolCompletionResponse};
|
||||||
|
|
||||||
|
/// A mock LLM provider that returns a predetermined result.
|
||||||
|
struct MockProvider {
|
||||||
|
name: String,
|
||||||
|
input_cost: Decimal,
|
||||||
|
output_cost: Decimal,
|
||||||
|
complete_result: Mutex<Option<Result<CompletionResponse, LlmError>>>,
|
||||||
|
tool_complete_result: Mutex<Option<Result<ToolCompletionResponse, LlmError>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MockProvider {
|
||||||
|
fn succeeding(name: &str, content: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
input_cost: Decimal::ZERO,
|
||||||
|
output_cost: Decimal::ZERO,
|
||||||
|
complete_result: Mutex::new(Some(Ok(CompletionResponse {
|
||||||
|
content: content.to_string(),
|
||||||
|
input_tokens: 10,
|
||||||
|
output_tokens: 5,
|
||||||
|
finish_reason: FinishReason::Stop,
|
||||||
|
response_id: None,
|
||||||
|
}))),
|
||||||
|
tool_complete_result: Mutex::new(Some(Ok(ToolCompletionResponse {
|
||||||
|
content: Some(content.to_string()),
|
||||||
|
tool_calls: vec![],
|
||||||
|
input_tokens: 10,
|
||||||
|
output_tokens: 5,
|
||||||
|
finish_reason: FinishReason::Stop,
|
||||||
|
response_id: None,
|
||||||
|
}))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn succeeding_with_cost(
|
||||||
|
name: &str,
|
||||||
|
content: &str,
|
||||||
|
input_cost: Decimal,
|
||||||
|
output_cost: Decimal,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
input_cost,
|
||||||
|
output_cost,
|
||||||
|
..Self::succeeding(name, content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failing_retryable(name: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
input_cost: Decimal::ZERO,
|
||||||
|
output_cost: Decimal::ZERO,
|
||||||
|
complete_result: Mutex::new(Some(Err(LlmError::RequestFailed {
|
||||||
|
provider: name.to_string(),
|
||||||
|
reason: "server error".to_string(),
|
||||||
|
}))),
|
||||||
|
tool_complete_result: Mutex::new(Some(Err(LlmError::RequestFailed {
|
||||||
|
provider: name.to_string(),
|
||||||
|
reason: "server error".to_string(),
|
||||||
|
}))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failing_non_retryable(name: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
input_cost: Decimal::ZERO,
|
||||||
|
output_cost: Decimal::ZERO,
|
||||||
|
complete_result: Mutex::new(Some(Err(LlmError::AuthFailed {
|
||||||
|
provider: name.to_string(),
|
||||||
|
}))),
|
||||||
|
tool_complete_result: Mutex::new(Some(Err(LlmError::AuthFailed {
|
||||||
|
provider: name.to_string(),
|
||||||
|
}))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn failing_rate_limited(name: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
input_cost: Decimal::ZERO,
|
||||||
|
output_cost: Decimal::ZERO,
|
||||||
|
complete_result: Mutex::new(Some(Err(LlmError::RateLimited {
|
||||||
|
provider: name.to_string(),
|
||||||
|
retry_after: Some(Duration::from_secs(30)),
|
||||||
|
}))),
|
||||||
|
tool_complete_result: Mutex::new(Some(Err(LlmError::RateLimited {
|
||||||
|
provider: name.to_string(),
|
||||||
|
retry_after: Some(Duration::from_secs(30)),
|
||||||
|
}))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl LlmProvider for MockProvider {
|
||||||
|
fn model_name(&self) -> &str {
|
||||||
|
&self.name
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||||
|
(self.input_cost, self.output_cost)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete(
|
||||||
|
&self,
|
||||||
|
_request: CompletionRequest,
|
||||||
|
) -> Result<CompletionResponse, LlmError> {
|
||||||
|
self.complete_result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.take()
|
||||||
|
.expect("MockProvider::complete called more than once")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn complete_with_tools(
|
||||||
|
&self,
|
||||||
|
_request: ToolCompletionRequest,
|
||||||
|
) -> Result<ToolCompletionResponse, LlmError> {
|
||||||
|
self.tool_complete_result
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.take()
|
||||||
|
.expect("MockProvider::complete_with_tools called more than once")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||||
|
Ok(vec![self.name.clone()])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_request() -> CompletionRequest {
|
||||||
|
CompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_tool_request() -> ToolCompletionRequest {
|
||||||
|
ToolCompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")], vec![])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 1: Primary succeeds, no failover occurs.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn primary_succeeds_no_failover() {
|
||||||
|
let primary = Arc::new(MockProvider::succeeding("primary", "primary response"));
|
||||||
|
let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
let response = failover.complete(make_request()).await.unwrap();
|
||||||
|
assert_eq!(response.content, "primary response");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 2: Primary fails with retryable error, fallback succeeds.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn primary_fails_retryable_fallback_succeeds() {
|
||||||
|
let primary = Arc::new(MockProvider::failing_retryable("primary"));
|
||||||
|
let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
let response = failover.complete(make_request()).await.unwrap();
|
||||||
|
assert_eq!(response.content, "fallback response");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 3: All providers fail, returns last error.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn all_providers_fail_returns_last_error() {
|
||||||
|
let primary = Arc::new(MockProvider::failing_retryable("primary"));
|
||||||
|
let fallback = Arc::new(MockProvider::failing_retryable("fallback"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
let err = failover.complete(make_request()).await.unwrap_err();
|
||||||
|
match err {
|
||||||
|
LlmError::RequestFailed { provider, .. } => {
|
||||||
|
assert_eq!(provider, "fallback");
|
||||||
|
}
|
||||||
|
other => panic!("expected RequestFailed, got: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 4: Non-retryable error fails immediately, no failover.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn non_retryable_error_fails_immediately() {
|
||||||
|
let primary = Arc::new(MockProvider::failing_non_retryable("primary"));
|
||||||
|
let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
let err = failover.complete(make_request()).await.unwrap_err();
|
||||||
|
match err {
|
||||||
|
LlmError::AuthFailed { provider } => {
|
||||||
|
assert_eq!(provider, "primary");
|
||||||
|
}
|
||||||
|
other => panic!("expected AuthFailed, got: {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test 5: Three providers, first two fail (retryable), third succeeds.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn three_providers_first_two_fail_third_succeeds() {
|
||||||
|
let p1 = Arc::new(MockProvider::failing_retryable("provider-1"));
|
||||||
|
let p2 = Arc::new(MockProvider::failing_rate_limited("provider-2"));
|
||||||
|
let p3 = Arc::new(MockProvider::succeeding("provider-3", "third time lucky"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![p1, p2, p3]).unwrap();
|
||||||
|
|
||||||
|
let response = failover.complete(make_request()).await.unwrap();
|
||||||
|
assert_eq!(response.content, "third time lucky");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test: complete_with_tools follows same failover logic.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn complete_with_tools_failover() {
|
||||||
|
let primary = Arc::new(MockProvider::failing_retryable("primary"));
|
||||||
|
let fallback = Arc::new(MockProvider::succeeding("fallback", "tools fallback"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
let response = failover
|
||||||
|
.complete_with_tools(make_tool_request())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(response.content.as_deref(), Some("tools fallback"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test: model_name and cost_per_token reflect the last-used provider.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn model_name_and_cost_track_last_used_provider() {
|
||||||
|
let fallback_cost = Decimal::new(15, 6); // 0.000015
|
||||||
|
|
||||||
|
let primary = Arc::new(MockProvider::failing_retryable("primary-model"));
|
||||||
|
let fallback = Arc::new(MockProvider::succeeding_with_cost(
|
||||||
|
"fallback-model",
|
||||||
|
"ok",
|
||||||
|
fallback_cost,
|
||||||
|
fallback_cost,
|
||||||
|
));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![primary, fallback]).unwrap();
|
||||||
|
|
||||||
|
// Before any call, defaults to primary (index 0).
|
||||||
|
assert_eq!(failover.model_name(), "primary-model");
|
||||||
|
assert_eq!(failover.cost_per_token(), (Decimal::ZERO, Decimal::ZERO));
|
||||||
|
|
||||||
|
// After failover, should reflect the fallback provider.
|
||||||
|
let _ = failover.complete(make_request()).await.unwrap();
|
||||||
|
assert_eq!(failover.model_name(), "fallback-model");
|
||||||
|
assert_eq!(failover.cost_per_token(), (fallback_cost, fallback_cost));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test: list_models aggregates from all providers.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn list_models_aggregates_all() {
|
||||||
|
let p1 = Arc::new(MockProvider::succeeding("model-a", "ok"));
|
||||||
|
let p2 = Arc::new(MockProvider::succeeding("model-b", "ok"));
|
||||||
|
|
||||||
|
let failover = FailoverProvider::new(vec![p1, p2]).unwrap();
|
||||||
|
|
||||||
|
let models = failover.list_models().await.unwrap();
|
||||||
|
assert!(models.contains(&"model-a".to_string()));
|
||||||
|
assert!(models.contains(&"model-b".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test: is_retryable correctly classifies errors.
|
||||||
|
#[test]
|
||||||
|
fn retryable_classification() {
|
||||||
|
// Retryable
|
||||||
|
assert!(is_retryable(&LlmError::RequestFailed {
|
||||||
|
provider: "p".into(),
|
||||||
|
reason: "err".into(),
|
||||||
|
}));
|
||||||
|
assert!(is_retryable(&LlmError::RateLimited {
|
||||||
|
provider: "p".into(),
|
||||||
|
retry_after: None,
|
||||||
|
}));
|
||||||
|
assert!(is_retryable(&LlmError::InvalidResponse {
|
||||||
|
provider: "p".into(),
|
||||||
|
reason: "bad json".into(),
|
||||||
|
}));
|
||||||
|
assert!(is_retryable(&LlmError::SessionRenewalFailed {
|
||||||
|
provider: "p".into(),
|
||||||
|
reason: "timeout".into(),
|
||||||
|
}));
|
||||||
|
assert!(is_retryable(&LlmError::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::ConnectionReset,
|
||||||
|
"reset"
|
||||||
|
))));
|
||||||
|
assert!(is_retryable(&LlmError::ModelNotAvailable {
|
||||||
|
provider: "p".into(),
|
||||||
|
model: "m".into(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Non-retryable
|
||||||
|
assert!(!is_retryable(&LlmError::AuthFailed {
|
||||||
|
provider: "p".into(),
|
||||||
|
}));
|
||||||
|
assert!(!is_retryable(&LlmError::SessionExpired {
|
||||||
|
provider: "p".into(),
|
||||||
|
}));
|
||||||
|
assert!(!is_retryable(&LlmError::ContextLengthExceeded {
|
||||||
|
used: 100_000,
|
||||||
|
limit: 50_000,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test: empty providers list returns error (not panic).
|
||||||
|
#[test]
|
||||||
|
fn empty_providers_returns_error() {
|
||||||
|
let result = FailoverProvider::new(vec![]);
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
+22
-12
@@ -8,13 +8,16 @@
|
|||||||
//! - **OpenAI-compatible**: Any endpoint that speaks the OpenAI API
|
//! - **OpenAI-compatible**: Any endpoint that speaks the OpenAI API
|
||||||
|
|
||||||
mod costs;
|
mod costs;
|
||||||
|
pub mod failover;
|
||||||
mod nearai;
|
mod nearai;
|
||||||
mod nearai_chat;
|
mod nearai_chat;
|
||||||
mod provider;
|
mod provider;
|
||||||
mod reasoning;
|
mod reasoning;
|
||||||
|
mod retry;
|
||||||
mod rig_adapter;
|
mod rig_adapter;
|
||||||
pub mod session;
|
pub mod session;
|
||||||
|
|
||||||
|
pub use failover::FailoverProvider;
|
||||||
pub use nearai::{ModelInfo, NearAiProvider};
|
pub use nearai::{ModelInfo, NearAiProvider};
|
||||||
pub use nearai_chat::NearAiChatProvider;
|
pub use nearai_chat::NearAiChatProvider;
|
||||||
pub use provider::{
|
pub use provider::{
|
||||||
@@ -33,7 +36,7 @@ use std::sync::Arc;
|
|||||||
use rig::client::CompletionClient;
|
use rig::client::CompletionClient;
|
||||||
use secrecy::ExposeSecret;
|
use secrecy::ExposeSecret;
|
||||||
|
|
||||||
use crate::config::{LlmBackend, LlmConfig, NearAiApiMode};
|
use crate::config::{LlmBackend, LlmConfig, NearAiApiMode, NearAiConfig};
|
||||||
use crate::error::LlmError;
|
use crate::error::LlmError;
|
||||||
|
|
||||||
/// Create an LLM provider based on configuration.
|
/// Create an LLM provider based on configuration.
|
||||||
@@ -46,7 +49,7 @@ pub fn create_llm_provider(
|
|||||||
session: Arc<SessionManager>,
|
session: Arc<SessionManager>,
|
||||||
) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
||||||
match config.backend {
|
match config.backend {
|
||||||
LlmBackend::NearAi => create_nearai_provider(config, session),
|
LlmBackend::NearAi => create_llm_provider_with_config(&config.nearai, session),
|
||||||
LlmBackend::OpenAi => create_openai_provider(config),
|
LlmBackend::OpenAi => create_openai_provider(config),
|
||||||
LlmBackend::Anthropic => create_anthropic_provider(config),
|
LlmBackend::Anthropic => create_anthropic_provider(config),
|
||||||
LlmBackend::Ollama => create_ollama_provider(config),
|
LlmBackend::Ollama => create_ollama_provider(config),
|
||||||
@@ -54,21 +57,28 @@ pub fn create_llm_provider(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_nearai_provider(
|
/// Create an LLM provider from a `NearAiConfig` directly.
|
||||||
config: &LlmConfig,
|
///
|
||||||
|
/// This is useful when constructing additional providers for failover,
|
||||||
|
/// where only the model name differs from the primary config.
|
||||||
|
pub fn create_llm_provider_with_config(
|
||||||
|
config: &NearAiConfig,
|
||||||
session: Arc<SessionManager>,
|
session: Arc<SessionManager>,
|
||||||
) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
||||||
match config.nearai.api_mode {
|
match config.api_mode {
|
||||||
NearAiApiMode::Responses => {
|
NearAiApiMode::Responses => {
|
||||||
tracing::info!("Using NEAR AI Responses API (chat-api) with session auth");
|
tracing::info!(
|
||||||
Ok(Arc::new(NearAiProvider::new(
|
model = %config.model,
|
||||||
config.nearai.clone(),
|
"Using Responses API (chat-api) with session auth"
|
||||||
session,
|
);
|
||||||
)))
|
Ok(Arc::new(NearAiProvider::new(config.clone(), session)))
|
||||||
}
|
}
|
||||||
NearAiApiMode::ChatCompletions => {
|
NearAiApiMode::ChatCompletions => {
|
||||||
tracing::info!("Using NEAR AI Chat Completions API (cloud-api) with API key auth");
|
tracing::info!(
|
||||||
Ok(Arc::new(NearAiChatProvider::new(config.nearai.clone())?))
|
model = %config.model,
|
||||||
|
"Using Chat Completions API (cloud-api) with API key auth"
|
||||||
|
);
|
||||||
|
Ok(Arc::new(NearAiChatProvider::new(config.clone())?))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+135
-93
@@ -19,6 +19,7 @@ use crate::llm::provider::{
|
|||||||
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall,
|
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall,
|
||||||
ToolCompletionRequest, ToolCompletionResponse,
|
ToolCompletionRequest, ToolCompletionResponse,
|
||||||
};
|
};
|
||||||
|
use crate::llm::retry::{is_retryable_status, retry_backoff_delay};
|
||||||
use crate::llm::session::SessionManager;
|
use crate::llm::session::SessionManager;
|
||||||
|
|
||||||
/// Information about an available model from NEAR AI API.
|
/// Information about an available model from NEAR AI API.
|
||||||
@@ -209,20 +210,20 @@ impl NearAiProvider {
|
|||||||
data: Option<Vec<ModelEntry>>,
|
data: Option<Vec<ModelEntry>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Ok(resp) = serde_json::from_str::<ModelsResponse>(&response_text) {
|
if let Ok(resp) = serde_json::from_str::<ModelsResponse>(&response_text)
|
||||||
if let Some(entries) = resp.models.or(resp.data) {
|
&& let Some(entries) = resp.models.or(resp.data)
|
||||||
let models: Vec<ModelInfo> = entries
|
{
|
||||||
.into_iter()
|
let models: Vec<ModelInfo> = entries
|
||||||
.filter_map(|e| {
|
.into_iter()
|
||||||
e.get_name().map(|name| ModelInfo {
|
.filter_map(|e| {
|
||||||
name,
|
e.get_name().map(|name| ModelInfo {
|
||||||
provider: None,
|
name,
|
||||||
})
|
provider: None,
|
||||||
})
|
})
|
||||||
.collect();
|
})
|
||||||
if !models.is_empty() {
|
.collect();
|
||||||
return Ok(models);
|
if !models.is_empty() {
|
||||||
}
|
return Ok(models);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,88 +271,139 @@ impl NearAiProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Inner request implementation without retry logic.
|
/// Inner request implementation with retry logic for transient errors.
|
||||||
|
///
|
||||||
|
/// Retries on HTTP 429, 500, 502, 503, 504 with exponential backoff.
|
||||||
|
/// Does not retry on client errors (400, 401, 403, 404) or parse errors.
|
||||||
async fn send_request_inner<T: Serialize + std::fmt::Debug, R: for<'de> Deserialize<'de>>(
|
async fn send_request_inner<T: Serialize + std::fmt::Debug, R: for<'de> Deserialize<'de>>(
|
||||||
&self,
|
&self,
|
||||||
path: &str,
|
path: &str,
|
||||||
body: &T,
|
body: &T,
|
||||||
) -> Result<R, LlmError> {
|
) -> Result<R, LlmError> {
|
||||||
let url = self.api_url(path);
|
let url = self.api_url(path);
|
||||||
let token = self.session.get_token().await?;
|
let max_retries = self.config.max_retries;
|
||||||
|
|
||||||
tracing::debug!("Sending request to NEAR AI: {}", url);
|
for attempt in 0..=max_retries {
|
||||||
tracing::debug!("Request body: {:?}", body);
|
let token = self.session.get_token().await?;
|
||||||
|
|
||||||
let response = self
|
tracing::debug!(
|
||||||
.client
|
"Sending request to NEAR AI: {} (attempt {})",
|
||||||
.post(&url)
|
url,
|
||||||
.header("Authorization", format!("Bearer {}", token.expose_secret()))
|
attempt + 1
|
||||||
.header("Content-Type", "application/json")
|
);
|
||||||
.json(body)
|
tracing::debug!("Request body: {:?}", body);
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| {
|
|
||||||
tracing::error!("NEAR AI request failed: {}", e);
|
|
||||||
e
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let status = response.status();
|
let response = self
|
||||||
let response_text = response.text().await.unwrap_or_default();
|
.client
|
||||||
|
.post(&url)
|
||||||
|
.header("Authorization", format!("Bearer {}", token.expose_secret()))
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.json(body)
|
||||||
|
.send()
|
||||||
|
.await;
|
||||||
|
|
||||||
tracing::debug!("NEAR AI response status: {}", status);
|
let response = match response {
|
||||||
tracing::debug!("NEAR AI response body: {}", response_text);
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
tracing::error!("NEAR AI request failed: {}", e);
|
||||||
|
// Network errors (timeout, connection refused) are transient
|
||||||
|
if attempt < max_retries {
|
||||||
|
let delay = retry_backoff_delay(attempt);
|
||||||
|
tracing::warn!(
|
||||||
|
"NEAR AI request error (attempt {}/{}), retrying in {:?}: {}",
|
||||||
|
attempt + 1,
|
||||||
|
max_retries + 1,
|
||||||
|
delay,
|
||||||
|
e,
|
||||||
|
);
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if !status.is_success() {
|
let status = response.status();
|
||||||
// Check for session expiration (401 with specific message patterns)
|
let response_text = response.text().await.unwrap_or_default();
|
||||||
if status.as_u16() == 401 {
|
|
||||||
let is_session_expired = response_text.to_lowercase().contains("session")
|
|
||||||
&& (response_text.to_lowercase().contains("expired")
|
|
||||||
|| response_text.to_lowercase().contains("invalid"));
|
|
||||||
|
|
||||||
if is_session_expired {
|
tracing::debug!("NEAR AI response status: {}", status);
|
||||||
return Err(LlmError::SessionExpired {
|
tracing::debug!("NEAR AI response body: {}", response_text);
|
||||||
|
|
||||||
|
if !status.is_success() {
|
||||||
|
let status_code = status.as_u16();
|
||||||
|
|
||||||
|
// Check for session expiration (401 with specific message patterns)
|
||||||
|
if status_code == 401 {
|
||||||
|
let lower = response_text.to_lowercase();
|
||||||
|
let is_session_expired = lower.contains("session")
|
||||||
|
&& (lower.contains("expired") || lower.contains("invalid"));
|
||||||
|
|
||||||
|
if is_session_expired {
|
||||||
|
return Err(LlmError::SessionExpired {
|
||||||
|
provider: "nearai".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generic 401 -- not retryable
|
||||||
|
return Err(LlmError::AuthFailed {
|
||||||
provider: "nearai".to_string(),
|
provider: "nearai".to_string(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generic 401 without session expiration indication
|
// Check if this is a transient error worth retrying
|
||||||
return Err(LlmError::AuthFailed {
|
if is_retryable_status(status_code) && attempt < max_retries {
|
||||||
provider: "nearai".to_string(),
|
let delay = retry_backoff_delay(attempt);
|
||||||
});
|
tracing::warn!(
|
||||||
}
|
"NEAR AI returned HTTP {} (attempt {}/{}), retrying in {:?}",
|
||||||
|
status_code,
|
||||||
|
attempt + 1,
|
||||||
|
max_retries + 1,
|
||||||
|
delay,
|
||||||
|
);
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
// Try to parse as JSON error
|
// Non-retryable error or exhausted retries
|
||||||
if let Ok(error) = serde_json::from_str::<NearAiErrorResponse>(&response_text) {
|
if let Ok(error) = serde_json::from_str::<NearAiErrorResponse>(&response_text) {
|
||||||
if status.as_u16() == 429 {
|
if status_code == 429 {
|
||||||
return Err(LlmError::RateLimited {
|
return Err(LlmError::RateLimited {
|
||||||
|
provider: "nearai".to_string(),
|
||||||
|
retry_after: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return Err(LlmError::RequestFailed {
|
||||||
provider: "nearai".to_string(),
|
provider: "nearai".to_string(),
|
||||||
retry_after: None,
|
reason: error.error,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return Err(LlmError::RequestFailed {
|
return Err(LlmError::RequestFailed {
|
||||||
provider: "nearai".to_string(),
|
provider: "nearai".to_string(),
|
||||||
reason: error.error,
|
reason: format!("HTTP {}: {}", status, response_text),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return Err(LlmError::RequestFailed {
|
// Success -- parse the response
|
||||||
provider: "nearai".to_string(),
|
return match serde_json::from_str::<R>(&response_text) {
|
||||||
reason: format!("HTTP {}: {}", status, response_text),
|
Ok(parsed) => Ok(parsed),
|
||||||
});
|
Err(e) => {
|
||||||
|
tracing::debug!("Response is not expected JSON format: {}", e);
|
||||||
|
tracing::debug!("Will try alternative parsing in caller");
|
||||||
|
Err(LlmError::InvalidResponse {
|
||||||
|
provider: "nearai".to_string(),
|
||||||
|
reason: format!("Parse error: {}. Raw: {}", e, response_text),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to parse as our expected type
|
// This is unreachable because the loop always returns, but the compiler
|
||||||
match serde_json::from_str::<R>(&response_text) {
|
// cannot prove that. Return a generic error as a safety net.
|
||||||
Ok(parsed) => Ok(parsed),
|
Err(LlmError::RequestFailed {
|
||||||
Err(e) => {
|
provider: "nearai".to_string(),
|
||||||
tracing::debug!("Response is not expected JSON format: {}", e);
|
reason: "retry loop exited unexpectedly".to_string(),
|
||||||
tracing::debug!("Will try alternative parsing in caller");
|
})
|
||||||
Err(LlmError::InvalidResponse {
|
|
||||||
provider: "nearai".to_string(),
|
|
||||||
reason: format!("Parse error: {}. Raw: {}", e, response_text),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -456,7 +508,7 @@ impl LlmProvider for NearAiProvider {
|
|||||||
Err(e) => return Err(e),
|
Err(e) => return Err(e),
|
||||||
};
|
};
|
||||||
|
|
||||||
tracing::debug!("NEAR AI response: {:?}", response);
|
tracing::debug!("NEAR AI response: output_items={}", response.output.len());
|
||||||
|
|
||||||
// Extract text from response output
|
// Extract text from response output
|
||||||
// Try multiple formats since API response shape may vary
|
// Try multiple formats since API response shape may vary
|
||||||
@@ -464,11 +516,6 @@ impl LlmProvider for NearAiProvider {
|
|||||||
.output
|
.output
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|item| {
|
.filter_map(|item| {
|
||||||
tracing::debug!(
|
|
||||||
"Processing output item: type={}, text={:?}",
|
|
||||||
item.item_type,
|
|
||||||
item.text
|
|
||||||
);
|
|
||||||
if item.item_type == "message" {
|
if item.item_type == "message" {
|
||||||
// First check for direct text field on item
|
// First check for direct text field on item
|
||||||
if let Some(ref text) = item.text {
|
if let Some(ref text) = item.text {
|
||||||
@@ -479,11 +526,6 @@ impl LlmProvider for NearAiProvider {
|
|||||||
contents
|
contents
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|c| {
|
.filter_map(|c| {
|
||||||
tracing::debug!(
|
|
||||||
"Content item: type={}, text={:?}",
|
|
||||||
c.content_type,
|
|
||||||
c.text
|
|
||||||
);
|
|
||||||
// Accept various content types that might contain text
|
// Accept various content types that might contain text
|
||||||
match c.content_type.as_str() {
|
match c.content_type.as_str() {
|
||||||
"output_text" | "text" => c.text.clone(),
|
"output_text" | "text" => c.text.clone(),
|
||||||
@@ -694,21 +736,21 @@ impl LlmProvider for NearAiProvider {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if item.item_type == "function_call" {
|
} else if item.item_type == "function_call"
|
||||||
if let (Some(name), Some(call_id)) = (&item.name, &item.call_id) {
|
&& let (Some(name), Some(call_id)) = (&item.name, &item.call_id)
|
||||||
// Parse arguments JSON string into Value
|
{
|
||||||
let arguments = item
|
// Parse arguments JSON string into Value
|
||||||
.arguments
|
let arguments = item
|
||||||
.as_ref()
|
.arguments
|
||||||
.and_then(|s| serde_json::from_str(s).ok())
|
.as_ref()
|
||||||
.unwrap_or(serde_json::Value::Object(Default::default()));
|
.and_then(|s| serde_json::from_str(s).ok())
|
||||||
|
.unwrap_or(serde_json::Value::Object(Default::default()));
|
||||||
|
|
||||||
tool_calls.push(ToolCall {
|
tool_calls.push(ToolCall {
|
||||||
id: call_id.clone(),
|
id: call_id.clone(),
|
||||||
name: name.clone(),
|
name: name.clone(),
|
||||||
arguments,
|
arguments,
|
||||||
});
|
});
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+95
-41
@@ -16,6 +16,7 @@ use crate::llm::provider::{
|
|||||||
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata,
|
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata,
|
||||||
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse,
|
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse,
|
||||||
};
|
};
|
||||||
|
use crate::llm::retry::{is_retryable_status, retry_backoff_delay};
|
||||||
|
|
||||||
/// NEAR AI Chat Completions API provider.
|
/// NEAR AI Chat Completions API provider.
|
||||||
pub struct NearAiChatProvider {
|
pub struct NearAiChatProvider {
|
||||||
@@ -62,63 +63,116 @@ impl NearAiChatProvider {
|
|||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send a request to the chat completions API.
|
/// Send a request to the chat completions API with retry on transient errors.
|
||||||
|
///
|
||||||
|
/// Retries on HTTP 429, 500, 502, 503, 504 with exponential backoff.
|
||||||
|
/// Does not retry on client errors (400, 401, 403, 404) or parse errors.
|
||||||
async fn send_request<T: Serialize, R: for<'de> Deserialize<'de>>(
|
async fn send_request<T: Serialize, R: for<'de> Deserialize<'de>>(
|
||||||
&self,
|
&self,
|
||||||
body: &T,
|
body: &T,
|
||||||
) -> Result<R, LlmError> {
|
) -> Result<R, LlmError> {
|
||||||
let url = self.api_url("chat/completions");
|
let url = self.api_url("chat/completions");
|
||||||
|
let max_retries = self.config.max_retries;
|
||||||
|
|
||||||
tracing::debug!("Sending request to NEAR AI Chat: {}", url);
|
for attempt in 0..=max_retries {
|
||||||
|
tracing::debug!(
|
||||||
|
"Sending request to NEAR AI Chat: {} (attempt {})",
|
||||||
|
url,
|
||||||
|
attempt + 1,
|
||||||
|
);
|
||||||
|
|
||||||
// Log the request body for debugging tool call issues
|
if tracing::enabled!(tracing::Level::DEBUG)
|
||||||
if let Ok(json) = serde_json::to_string(body) {
|
&& let Ok(json) = serde_json::to_string(body)
|
||||||
tracing::debug!("NEAR AI Chat request body: {}", json);
|
{
|
||||||
}
|
tracing::debug!("NEAR AI Chat request body: {}", json);
|
||||||
|
}
|
||||||
|
|
||||||
let response = self
|
let response = self
|
||||||
.client
|
.client
|
||||||
.post(&url)
|
.post(&url)
|
||||||
.header("Authorization", format!("Bearer {}", self.api_key()))
|
.header("Authorization", format!("Bearer {}", self.api_key()))
|
||||||
.header("Content-Type", "application/json")
|
.header("Content-Type", "application/json")
|
||||||
.json(body)
|
.json(body)
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await;
|
||||||
.map_err(|e| {
|
|
||||||
tracing::error!("NEAR AI Chat request failed: {}", e);
|
let response = match response {
|
||||||
LlmError::RequestFailed {
|
Ok(r) => r,
|
||||||
provider: "nearai_chat".to_string(),
|
Err(e) => {
|
||||||
reason: e.to_string(),
|
tracing::error!("NEAR AI Chat request failed: {}", e);
|
||||||
|
if attempt < max_retries {
|
||||||
|
let delay = retry_backoff_delay(attempt);
|
||||||
|
tracing::warn!(
|
||||||
|
"NEAR AI Chat request error (attempt {}/{}), retrying in {:?}: {}",
|
||||||
|
attempt + 1,
|
||||||
|
max_retries + 1,
|
||||||
|
delay,
|
||||||
|
e,
|
||||||
|
);
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Err(LlmError::RequestFailed {
|
||||||
|
provider: "nearai_chat".to_string(),
|
||||||
|
reason: e.to_string(),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
})?;
|
};
|
||||||
|
|
||||||
let status = response.status();
|
let status = response.status();
|
||||||
let response_text = response.text().await.unwrap_or_default();
|
let response_text = response.text().await.unwrap_or_default();
|
||||||
|
|
||||||
tracing::debug!("NEAR AI Chat response status: {}", status);
|
tracing::debug!("NEAR AI Chat response status: {}", status);
|
||||||
tracing::debug!("NEAR AI Chat response body: {}", response_text);
|
tracing::debug!("NEAR AI Chat response body: {}", response_text);
|
||||||
|
|
||||||
if !status.is_success() {
|
if !status.is_success() {
|
||||||
if status.as_u16() == 401 {
|
let status_code = status.as_u16();
|
||||||
return Err(LlmError::AuthFailed {
|
|
||||||
|
// Auth errors are not retryable
|
||||||
|
if status_code == 401 {
|
||||||
|
return Err(LlmError::AuthFailed {
|
||||||
|
provider: "nearai_chat".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Transient errors: retry with backoff
|
||||||
|
if is_retryable_status(status_code) && attempt < max_retries {
|
||||||
|
let delay = retry_backoff_delay(attempt);
|
||||||
|
tracing::warn!(
|
||||||
|
"NEAR AI Chat returned HTTP {} (attempt {}/{}), retrying in {:?}",
|
||||||
|
status_code,
|
||||||
|
attempt + 1,
|
||||||
|
max_retries + 1,
|
||||||
|
delay,
|
||||||
|
);
|
||||||
|
tokio::time::sleep(delay).await;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-retryable or exhausted retries
|
||||||
|
if status_code == 429 {
|
||||||
|
return Err(LlmError::RateLimited {
|
||||||
|
provider: "nearai_chat".to_string(),
|
||||||
|
retry_after: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return Err(LlmError::RequestFailed {
|
||||||
provider: "nearai_chat".to_string(),
|
provider: "nearai_chat".to_string(),
|
||||||
|
reason: format!("HTTP {}: {}", status, response_text),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if status.as_u16() == 429 {
|
|
||||||
return Err(LlmError::RateLimited {
|
// Success — parse the response
|
||||||
provider: "nearai_chat".to_string(),
|
return serde_json::from_str(&response_text).map_err(|e| LlmError::InvalidResponse {
|
||||||
retry_after: None,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
return Err(LlmError::RequestFailed {
|
|
||||||
provider: "nearai_chat".to_string(),
|
provider: "nearai_chat".to_string(),
|
||||||
reason: format!("HTTP {}: {}", status, response_text),
|
reason: format!("JSON parse error: {}. Raw: {}", e, response_text),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
serde_json::from_str(&response_text).map_err(|e| LlmError::InvalidResponse {
|
// Safety net: unreachable because the loop always returns
|
||||||
|
Err(LlmError::RequestFailed {
|
||||||
provider: "nearai_chat".to_string(),
|
provider: "nearai_chat".to_string(),
|
||||||
reason: format!("JSON parse error: {}. Raw: {}", e, response_text),
|
reason: "retry loop exited unexpectedly".to_string(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -395,10 +449,10 @@ fn flatten_tool_messages(messages: Vec<ChatCompletionMessage>) -> Vec<ChatComple
|
|||||||
if let (true, Some(calls)) = (msg.role == "assistant", &msg.tool_calls) {
|
if let (true, Some(calls)) = (msg.role == "assistant", &msg.tool_calls) {
|
||||||
// Convert assistant tool_calls into descriptive text
|
// Convert assistant tool_calls into descriptive text
|
||||||
let mut parts: Vec<String> = Vec::new();
|
let mut parts: Vec<String> = Vec::new();
|
||||||
if let Some(ref text) = msg.content {
|
if let Some(ref text) = msg.content
|
||||||
if !text.is_empty() {
|
&& !text.is_empty()
|
||||||
parts.push(text.clone());
|
{
|
||||||
}
|
parts.push(text.clone());
|
||||||
}
|
}
|
||||||
for tc in calls {
|
for tc in calls {
|
||||||
parts.push(format!(
|
parts.push(format!(
|
||||||
|
|||||||
+21
-15
@@ -113,6 +113,12 @@ pub struct ToolSelection {
|
|||||||
pub reasoning: String,
|
pub reasoning: String,
|
||||||
/// Alternative tools considered.
|
/// Alternative tools considered.
|
||||||
pub alternatives: Vec<String>,
|
pub alternatives: Vec<String>,
|
||||||
|
/// The tool call ID from the LLM response.
|
||||||
|
///
|
||||||
|
/// OpenAI-compatible providers assign each tool call a unique ID that must
|
||||||
|
/// be echoed back in the corresponding tool result message. Without this,
|
||||||
|
/// the provider cannot match results to their originating calls.
|
||||||
|
pub tool_call_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Token usage from a single LLM call.
|
/// Token usage from a single LLM call.
|
||||||
@@ -244,6 +250,7 @@ impl Reasoning {
|
|||||||
parameters: tool_call.arguments,
|
parameters: tool_call.arguments,
|
||||||
reasoning: reasoning.clone(),
|
reasoning: reasoning.clone(),
|
||||||
alternatives: vec![],
|
alternatives: vec![],
|
||||||
|
tool_call_id: tool_call.id,
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
@@ -581,21 +588,20 @@ fn recover_tool_calls_from_content(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Try JSON first: {"name":"x","arguments":{}}
|
// Try JSON first: {"name":"x","arguments":{}}
|
||||||
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(inner) {
|
if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(inner)
|
||||||
if let Some(name) = parsed.get("name").and_then(|v| v.as_str()) {
|
&& let Some(name) = parsed.get("name").and_then(|v| v.as_str())
|
||||||
if tool_names.contains(name) {
|
&& tool_names.contains(name)
|
||||||
let arguments = parsed
|
{
|
||||||
.get("arguments")
|
let arguments = parsed
|
||||||
.cloned()
|
.get("arguments")
|
||||||
.unwrap_or(serde_json::Value::Object(Default::default()));
|
.cloned()
|
||||||
calls.push(ToolCall {
|
.unwrap_or(serde_json::Value::Object(Default::default()));
|
||||||
id: format!("recovered_{}", calls.len()),
|
calls.push(ToolCall {
|
||||||
name: name.to_string(),
|
id: format!("recovered_{}", calls.len()),
|
||||||
arguments,
|
name: name.to_string(),
|
||||||
});
|
arguments,
|
||||||
continue;
|
});
|
||||||
}
|
continue;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Bare tool name (e.g. "<tool_call>tool_list</tool_call>")
|
// Bare tool name (e.g. "<tool_call>tool_list</tool_call>")
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
//! Shared retry helpers for LLM providers.
|
||||||
|
//!
|
||||||
|
//! Provides exponential backoff with jitter and retryable status classification
|
||||||
|
//! used by both `NearAiProvider` and `NearAiChatProvider`.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use rand::Rng;
|
||||||
|
|
||||||
|
/// Returns `true` if the HTTP status code is transient and worth retrying.
|
||||||
|
pub(crate) fn is_retryable_status(status: u16) -> bool {
|
||||||
|
matches!(status, 429 | 500 | 502 | 503 | 504)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Calculate exponential backoff delay with random jitter.
|
||||||
|
///
|
||||||
|
/// Base delay is 1 second, doubled each attempt, with +/-25% jitter.
|
||||||
|
/// - attempt 0: ~1s (0.75s - 1.25s)
|
||||||
|
/// - attempt 1: ~2s (1.5s - 2.5s)
|
||||||
|
/// - attempt 2: ~4s (3.0s - 5.0s)
|
||||||
|
pub(crate) fn retry_backoff_delay(attempt: u32) -> Duration {
|
||||||
|
let base_ms: u64 = 1000u64.saturating_mul(2u64.saturating_pow(attempt));
|
||||||
|
let jitter_range = base_ms / 4; // 25%
|
||||||
|
let jitter = if jitter_range > 0 {
|
||||||
|
let offset = rand::thread_rng().gen_range(0..=jitter_range * 2);
|
||||||
|
offset as i64 - jitter_range as i64
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let delay_ms = (base_ms as i64 + jitter).max(100) as u64;
|
||||||
|
Duration::from_millis(delay_ms)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_retryable_status() {
|
||||||
|
// Transient errors should be retryable
|
||||||
|
assert!(is_retryable_status(429));
|
||||||
|
assert!(is_retryable_status(500));
|
||||||
|
assert!(is_retryable_status(502));
|
||||||
|
assert!(is_retryable_status(503));
|
||||||
|
assert!(is_retryable_status(504));
|
||||||
|
|
||||||
|
// Client errors should not be retryable
|
||||||
|
assert!(!is_retryable_status(400));
|
||||||
|
assert!(!is_retryable_status(401));
|
||||||
|
assert!(!is_retryable_status(403));
|
||||||
|
assert!(!is_retryable_status(404));
|
||||||
|
assert!(!is_retryable_status(422));
|
||||||
|
|
||||||
|
// Success codes should not be retryable
|
||||||
|
assert!(!is_retryable_status(200));
|
||||||
|
assert!(!is_retryable_status(201));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_retry_backoff_delay_exponential_growth() {
|
||||||
|
// Run multiple samples to verify the range, accounting for jitter
|
||||||
|
for _ in 0..20 {
|
||||||
|
let d0 = retry_backoff_delay(0);
|
||||||
|
let d1 = retry_backoff_delay(1);
|
||||||
|
let d2 = retry_backoff_delay(2);
|
||||||
|
|
||||||
|
// Attempt 0: base 1000ms, jitter +/-250ms -> [750, 1250]
|
||||||
|
assert!(d0.as_millis() >= 750, "attempt 0 too low: {:?}", d0);
|
||||||
|
assert!(d0.as_millis() <= 1250, "attempt 0 too high: {:?}", d0);
|
||||||
|
|
||||||
|
// Attempt 1: base 2000ms, jitter +/-500ms -> [1500, 2500]
|
||||||
|
assert!(d1.as_millis() >= 1500, "attempt 1 too low: {:?}", d1);
|
||||||
|
assert!(d1.as_millis() <= 2500, "attempt 1 too high: {:?}", d1);
|
||||||
|
|
||||||
|
// Attempt 2: base 4000ms, jitter +/-1000ms -> [3000, 5000]
|
||||||
|
assert!(d2.as_millis() >= 3000, "attempt 2 too low: {:?}", d2);
|
||||||
|
assert!(d2.as_millis() <= 5000, "attempt 2 too high: {:?}", d2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_retry_backoff_delay_minimum() {
|
||||||
|
// Even at attempt 0, delay should be at least 100ms (the minimum floor)
|
||||||
|
for _ in 0..20 {
|
||||||
|
let delay = retry_backoff_delay(0);
|
||||||
|
assert!(delay.as_millis() >= 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_retry_backoff_delay_no_overflow() {
|
||||||
|
// Very high attempt numbers should not panic from overflow
|
||||||
|
let delay = retry_backoff_delay(30);
|
||||||
|
assert!(delay.as_millis() >= 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
+37
-182
@@ -31,8 +31,6 @@ pub struct SessionConfig {
|
|||||||
pub auth_base_url: String,
|
pub auth_base_url: String,
|
||||||
/// Path to session file (e.g., ~/.ironclaw/session.json).
|
/// Path to session file (e.g., ~/.ironclaw/session.json).
|
||||||
pub session_path: PathBuf,
|
pub session_path: PathBuf,
|
||||||
/// Port range for OAuth callback server.
|
|
||||||
pub callback_port_range: (u16, u16),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for SessionConfig {
|
impl Default for SessionConfig {
|
||||||
@@ -40,7 +38,6 @@ impl Default for SessionConfig {
|
|||||||
Self {
|
Self {
|
||||||
auth_base_url: "https://private.near.ai".to_string(),
|
auth_base_url: "https://private.near.ai".to_string(),
|
||||||
session_path: default_session_path(),
|
session_path: default_session_path(),
|
||||||
callback_port_range: (9876, 9886),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -62,7 +59,7 @@ pub struct SessionManager {
|
|||||||
/// Prevents thundering herd during concurrent 401s.
|
/// Prevents thundering herd during concurrent 401s.
|
||||||
renewal_lock: Mutex<()>,
|
renewal_lock: Mutex<()>,
|
||||||
/// Optional database store for persisting session to the settings table.
|
/// Optional database store for persisting session to the settings table.
|
||||||
store: RwLock<Option<Arc<crate::history::Store>>>,
|
store: RwLock<Option<Arc<dyn crate::db::Database>>>,
|
||||||
/// User ID for DB settings (default: "default").
|
/// User ID for DB settings (default: "default").
|
||||||
user_id: RwLock<String>,
|
user_id: RwLock<String>,
|
||||||
}
|
}
|
||||||
@@ -83,16 +80,16 @@ impl SessionManager {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Try to load existing session synchronously during construction
|
// Try to load existing session synchronously during construction
|
||||||
if let Ok(data) = std::fs::read_to_string(&manager.config.session_path) {
|
if let Ok(data) = std::fs::read_to_string(&manager.config.session_path)
|
||||||
if let Ok(session) = serde_json::from_str::<SessionData>(&data) {
|
&& let Ok(session) = serde_json::from_str::<SessionData>(&data)
|
||||||
// We can't await here, so we use try_write
|
{
|
||||||
if let Ok(mut guard) = manager.token.try_write() {
|
// We can't await here, so we use try_write
|
||||||
*guard = Some(SecretString::from(session.session_token));
|
if let Ok(mut guard) = manager.token.try_write() {
|
||||||
tracing::info!(
|
*guard = Some(SecretString::from(session.session_token));
|
||||||
"Loaded session token from {}",
|
tracing::info!(
|
||||||
manager.config.session_path.display()
|
"Loaded session token from {}",
|
||||||
);
|
manager.config.session_path.display()
|
||||||
}
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,7 +122,7 @@ impl SessionManager {
|
|||||||
/// When a store is attached, session tokens are saved to the `settings`
|
/// When a store is attached, session tokens are saved to the `settings`
|
||||||
/// table (key: `nearai.session_token`) in addition to the disk file.
|
/// table (key: `nearai.session_token`) in addition to the disk file.
|
||||||
/// On load, DB is preferred over disk.
|
/// On load, DB is preferred over disk.
|
||||||
pub async fn attach_store(&self, store: Arc<crate::history::Store>, user_id: &str) {
|
pub async fn attach_store(&self, store: Arc<dyn crate::db::Database>, user_id: &str) {
|
||||||
*self.store.write().await = Some(store);
|
*self.store.write().await = Some(store);
|
||||||
*self.user_id.write().await = user_id.to_string();
|
*self.user_id.write().await = user_id.to_string();
|
||||||
|
|
||||||
@@ -222,38 +219,21 @@ impl SessionManager {
|
|||||||
|
|
||||||
/// Start the OAuth login flow.
|
/// Start the OAuth login flow.
|
||||||
///
|
///
|
||||||
/// 1. Find an available port for the callback server
|
/// 1. Bind the fixed callback port
|
||||||
/// 2. Print the auth URL and attempt to open browser
|
/// 2. Print the auth URL and attempt to open browser
|
||||||
/// 3. Wait for OAuth callback with session token
|
/// 3. Wait for OAuth callback with session token
|
||||||
/// 4. Save and return the token
|
/// 4. Save and return the token
|
||||||
async fn initiate_login(&self) -> Result<(), LlmError> {
|
async fn initiate_login(&self) -> Result<(), LlmError> {
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
use crate::cli::oauth_defaults::{self, OAUTH_CALLBACK_PORT};
|
||||||
use tokio::net::TcpListener;
|
|
||||||
|
|
||||||
// Find an available port
|
let listener = oauth_defaults::bind_callback_listener()
|
||||||
let mut listener = None;
|
.await
|
||||||
let mut port = 0;
|
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||||
|
provider: "nearai".to_string(),
|
||||||
|
reason: e.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
for p in self.config.callback_port_range.0..=self.config.callback_port_range.1 {
|
let callback_url = format!("http://127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||||
match TcpListener::bind(format!("127.0.0.1:{}", p)).await {
|
|
||||||
Ok(l) => {
|
|
||||||
listener = Some(l);
|
|
||||||
port = p;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(_) => continue,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let listener = listener.ok_or_else(|| LlmError::SessionRenewalFailed {
|
|
||||||
provider: "nearai".to_string(),
|
|
||||||
reason: format!(
|
|
||||||
"Could not find available port in range {}-{}",
|
|
||||||
self.config.callback_port_range.0, self.config.callback_port_range.1
|
|
||||||
),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let callback_url = format!("http://127.0.0.1:{}", port);
|
|
||||||
|
|
||||||
// Show auth provider menu
|
// Show auth provider menu
|
||||||
println!();
|
println!();
|
||||||
@@ -333,138 +313,16 @@ impl SessionManager {
|
|||||||
println!();
|
println!();
|
||||||
println!("Waiting for authentication...");
|
println!("Waiting for authentication...");
|
||||||
|
|
||||||
// Wait for callback with timeout
|
// The NEAR AI API redirects to: {frontend_callback}/auth/callback?token=X&...
|
||||||
// The API redirects to: {frontend_callback}/auth/callback?token=X&session_id=X&expires_at=X&is_new_user=X
|
let session_token =
|
||||||
let timeout = std::time::Duration::from_secs(300); // 5 minutes
|
oauth_defaults::wait_for_callback(listener, "/auth/callback", "token", "NEAR AI")
|
||||||
let selected_provider = auth_provider.to_string();
|
.await
|
||||||
let (session_token, auth_provider) = tokio::time::timeout(timeout, async move {
|
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||||
loop {
|
provider: "nearai".to_string(),
|
||||||
let (mut socket, _) = listener.accept().await.map_err(|e| {
|
reason: e.to_string(),
|
||||||
LlmError::SessionRenewalFailed {
|
|
||||||
provider: "nearai".to_string(),
|
|
||||||
reason: format!("Failed to accept connection: {}", e),
|
|
||||||
}
|
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let mut reader = BufReader::new(&mut socket);
|
let auth_provider = Some(auth_provider.to_string());
|
||||||
let mut request_line = String::new();
|
|
||||||
reader.read_line(&mut request_line).await.map_err(|e| {
|
|
||||||
LlmError::SessionRenewalFailed {
|
|
||||||
provider: "nearai".to_string(),
|
|
||||||
reason: format!("Failed to read request: {}", e),
|
|
||||||
}
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// Parse GET /auth/callback?token=xxx&session_id=xxx&expires_at=xxx&is_new_user=xxx HTTP/1.1
|
|
||||||
if let Some(path) = request_line.split_whitespace().nth(1) {
|
|
||||||
if path.starts_with("/auth/callback") {
|
|
||||||
// Parse query parameters
|
|
||||||
if let Some(query) = path.split('?').nth(1) {
|
|
||||||
let mut token = None;
|
|
||||||
|
|
||||||
for param in query.split('&') {
|
|
||||||
let parts: Vec<&str> = param.splitn(2, '=').collect();
|
|
||||||
if parts.len() == 2 && parts[0] == "token" {
|
|
||||||
token = Some(
|
|
||||||
urlencoding::decode(parts[1])
|
|
||||||
.unwrap_or_else(|_| parts[1].into())
|
|
||||||
.into_owned(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(token) = token {
|
|
||||||
// Send success response with nice styling
|
|
||||||
let response = concat!(
|
|
||||||
"HTTP/1.1 200 OK\r\n",
|
|
||||||
"Content-Type: text/html; charset=utf-8\r\n",
|
|
||||||
"Connection: close\r\n",
|
|
||||||
"\r\n",
|
|
||||||
"<!DOCTYPE html>\n",
|
|
||||||
"<html>\n",
|
|
||||||
"<head>\n",
|
|
||||||
" <meta charset=\"utf-8\">\n",
|
|
||||||
" <title>NEAR AI - Authentication Successful</title>\n",
|
|
||||||
" <style>\n",
|
|
||||||
" * { margin: 0; padding: 0; box-sizing: border-box; }\n",
|
|
||||||
" body {\n",
|
|
||||||
" font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;\n",
|
|
||||||
" background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);\n",
|
|
||||||
" min-height: 100vh;\n",
|
|
||||||
" display: flex;\n",
|
|
||||||
" align-items: center;\n",
|
|
||||||
" justify-content: center;\n",
|
|
||||||
" color: #fff;\n",
|
|
||||||
" }\n",
|
|
||||||
" .container {\n",
|
|
||||||
" text-align: center;\n",
|
|
||||||
" padding: 3rem;\n",
|
|
||||||
" background: rgba(255,255,255,0.05);\n",
|
|
||||||
" border-radius: 16px;\n",
|
|
||||||
" backdrop-filter: blur(10px);\n",
|
|
||||||
" border: 1px solid rgba(255,255,255,0.1);\n",
|
|
||||||
" max-width: 400px;\n",
|
|
||||||
" }\n",
|
|
||||||
" .checkmark {\n",
|
|
||||||
" width: 80px;\n",
|
|
||||||
" height: 80px;\n",
|
|
||||||
" background: linear-gradient(135deg, #00d9a5 0%, #00b386 100%);\n",
|
|
||||||
" border-radius: 50%;\n",
|
|
||||||
" display: flex;\n",
|
|
||||||
" align-items: center;\n",
|
|
||||||
" justify-content: center;\n",
|
|
||||||
" margin: 0 auto 1.5rem;\n",
|
|
||||||
" font-size: 40px;\n",
|
|
||||||
" }\n",
|
|
||||||
" h1 {\n",
|
|
||||||
" font-size: 1.5rem;\n",
|
|
||||||
" font-weight: 600;\n",
|
|
||||||
" margin-bottom: 0.75rem;\n",
|
|
||||||
" }\n",
|
|
||||||
" p {\n",
|
|
||||||
" color: rgba(255,255,255,0.7);\n",
|
|
||||||
" font-size: 0.95rem;\n",
|
|
||||||
" line-height: 1.5;\n",
|
|
||||||
" }\n",
|
|
||||||
" .brand {\n",
|
|
||||||
" margin-top: 2rem;\n",
|
|
||||||
" padding-top: 1.5rem;\n",
|
|
||||||
" border-top: 1px solid rgba(255,255,255,0.1);\n",
|
|
||||||
" font-size: 0.8rem;\n",
|
|
||||||
" color: rgba(255,255,255,0.4);\n",
|
|
||||||
" }\n",
|
|
||||||
" </style>\n",
|
|
||||||
"</head>\n",
|
|
||||||
"<body>\n",
|
|
||||||
" <div class=\"container\">\n",
|
|
||||||
" <div class=\"checkmark\">✓</div>\n",
|
|
||||||
" <h1>Authentication Successful</h1>\n",
|
|
||||||
" <p>You can close this window and return to the terminal.</p>\n",
|
|
||||||
" <div class=\"brand\">NEAR AI Agent</div>\n",
|
|
||||||
" </div>\n",
|
|
||||||
"</body>\n",
|
|
||||||
"</html>"
|
|
||||||
);
|
|
||||||
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
let _ = socket.shutdown().await;
|
|
||||||
|
|
||||||
return Ok::<_, LlmError>((token, Some(selected_provider.clone())));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Not the callback we're looking for, send 404
|
|
||||||
let response = "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n";
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|_| LlmError::SessionRenewalFailed {
|
|
||||||
provider: "nearai".to_string(),
|
|
||||||
reason: "Authentication timed out after 5 minutes".to_string(),
|
|
||||||
})??;
|
|
||||||
|
|
||||||
// Save the token
|
// Save the token
|
||||||
self.save_session(&session_token, auth_provider.as_deref())
|
self.save_session(&session_token, auth_provider.as_deref())
|
||||||
@@ -642,15 +500,14 @@ pub async fn create_session_manager(config: SessionConfig) -> Arc<SessionManager
|
|||||||
let manager = SessionManager::new_async(config).await;
|
let manager = SessionManager::new_async(config).await;
|
||||||
|
|
||||||
// Check for legacy env var and migrate if present and no file token
|
// Check for legacy env var and migrate if present and no file token
|
||||||
if !manager.has_token().await {
|
if !manager.has_token().await
|
||||||
if let Ok(token) = std::env::var("NEARAI_SESSION_TOKEN") {
|
&& let Ok(token) = std::env::var("NEARAI_SESSION_TOKEN")
|
||||||
if !token.is_empty() {
|
&& !token.is_empty()
|
||||||
tracing::info!("Migrating session token from NEARAI_SESSION_TOKEN env var to file");
|
{
|
||||||
manager.set_token(SecretString::from(token.clone())).await;
|
tracing::info!("Migrating session token from NEARAI_SESSION_TOKEN env var to file");
|
||||||
if let Err(e) = manager.save_session(&token, None).await {
|
manager.set_token(SecretString::from(token.clone())).await;
|
||||||
tracing::warn!("Failed to save migrated session: {}", e);
|
if let Err(e) = manager.save_session(&token, None).await {
|
||||||
}
|
tracing::warn!("Failed to save migrated session: {}", e);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -671,7 +528,6 @@ mod tests {
|
|||||||
let config = SessionConfig {
|
let config = SessionConfig {
|
||||||
auth_base_url: "https://example.com".to_string(),
|
auth_base_url: "https://example.com".to_string(),
|
||||||
session_path: session_path.clone(),
|
session_path: session_path.clone(),
|
||||||
callback_port_range: (9900, 9910),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let manager = SessionManager::new_async(config.clone()).await;
|
let manager = SessionManager::new_async(config.clone()).await;
|
||||||
@@ -712,7 +568,6 @@ mod tests {
|
|||||||
let config = SessionConfig {
|
let config = SessionConfig {
|
||||||
auth_base_url: "https://example.com".to_string(),
|
auth_base_url: "https://example.com".to_string(),
|
||||||
session_path: dir.path().join("nonexistent.json"),
|
session_path: dir.path().join("nonexistent.json"),
|
||||||
callback_port_range: (9900, 9910),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let manager = SessionManager::new_async(config).await;
|
let manager = SessionManager::new_async(config).await;
|
||||||
|
|||||||
+238
-118
@@ -17,22 +17,22 @@ use ironclaw::{
|
|||||||
web::log_layer::{LogBroadcaster, WebLogLayer},
|
web::log_layer::{LogBroadcaster, WebLogLayer},
|
||||||
},
|
},
|
||||||
cli::{
|
cli::{
|
||||||
Cli, Command, run_mcp_command, run_memory_command, run_pairing_command, run_status_command,
|
Cli, Command, run_mcp_command, run_pairing_command, run_status_command, run_tool_command,
|
||||||
run_tool_command,
|
|
||||||
},
|
},
|
||||||
config::Config,
|
config::Config,
|
||||||
context::ContextManager,
|
context::ContextManager,
|
||||||
extensions::ExtensionManager,
|
extensions::ExtensionManager,
|
||||||
history::Store,
|
llm::{
|
||||||
llm::{SessionConfig, create_llm_provider, create_session_manager},
|
FailoverProvider, LlmProvider, SessionConfig, create_llm_provider,
|
||||||
|
create_llm_provider_with_config, create_session_manager,
|
||||||
|
},
|
||||||
orchestrator::{
|
orchestrator::{
|
||||||
ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore,
|
ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore,
|
||||||
api::OrchestratorState,
|
api::OrchestratorState,
|
||||||
},
|
},
|
||||||
pairing::PairingStore,
|
pairing::PairingStore,
|
||||||
safety::SafetyLayer,
|
safety::SafetyLayer,
|
||||||
secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore},
|
secrets::SecretsStore,
|
||||||
setup::{SetupConfig, SetupWizard},
|
|
||||||
tools::{
|
tools::{
|
||||||
ToolRegistry,
|
ToolRegistry,
|
||||||
mcp::{McpClient, McpSessionManager, config::load_mcp_servers_from_db, is_authenticated},
|
mcp::{McpClient, McpSessionManager, config::load_mcp_servers_from_db, is_authenticated},
|
||||||
@@ -41,6 +41,14 @@ use ironclaw::{
|
|||||||
workspace::{EmbeddingProvider, NearAiEmbeddings, OpenAiEmbeddings, Workspace},
|
workspace::{EmbeddingProvider, NearAiEmbeddings, OpenAiEmbeddings, Workspace},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
use ironclaw::secrets::LibSqlSecretsStore;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
use ironclaw::secrets::PostgresSecretsStore;
|
||||||
|
use ironclaw::secrets::SecretsCrypto;
|
||||||
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
|
use ironclaw::setup::{SetupConfig, SetupWizard};
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
@@ -85,18 +93,14 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
.init();
|
.init();
|
||||||
|
|
||||||
// Memory commands need database (and optionally embeddings)
|
// Memory commands need database (and optionally embeddings)
|
||||||
let _ = dotenvy::dotenv();
|
|
||||||
let config = Config::from_env()
|
let config = Config::from_env()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
let store = ironclaw::history::Store::new(&config.database).await?;
|
|
||||||
store.run_migrations().await?;
|
|
||||||
|
|
||||||
// Set up embeddings if available
|
// Set up embeddings if available
|
||||||
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
||||||
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
||||||
session_path: config.llm.nearai.session_path.clone(),
|
session_path: config.llm.nearai.session_path.clone(),
|
||||||
..Default::default()
|
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -130,7 +134,14 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
return run_memory_command(mem_cmd.clone(), store.pool(), embeddings).await;
|
// Create a Database-trait-backed workspace for the memory command
|
||||||
|
let db: Arc<dyn ironclaw::db::Database> =
|
||||||
|
ironclaw::db::connect_from_config(&config.database)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
|
||||||
|
return ironclaw::cli::run_memory_command_with_db(mem_cmd.clone(), db, embeddings)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
Some(Command::Pairing(pairing_cmd)) => {
|
Some(Command::Pairing(pairing_cmd)) => {
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
@@ -142,7 +153,6 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
return run_pairing_command(pairing_cmd.clone()).map_err(|e| anyhow::anyhow!("{}", e));
|
return run_pairing_command(pairing_cmd.clone()).map_err(|e| anyhow::anyhow!("{}", e));
|
||||||
}
|
}
|
||||||
Some(Command::Status) => {
|
Some(Command::Status) => {
|
||||||
let _ = dotenvy::dotenv();
|
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
.with_env_filter(
|
.with_env_filter(
|
||||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")),
|
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")),
|
||||||
@@ -210,15 +220,13 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
model
|
model
|
||||||
);
|
);
|
||||||
|
|
||||||
// Load allowed tools from config (env var or defaults).
|
|
||||||
let claude_config = ironclaw::config::ClaudeCodeConfig::from_env();
|
|
||||||
let config = ironclaw::worker::claude_bridge::ClaudeBridgeConfig {
|
let config = ironclaw::worker::claude_bridge::ClaudeBridgeConfig {
|
||||||
job_id: *job_id,
|
job_id: *job_id,
|
||||||
orchestrator_url: orchestrator_url.clone(),
|
orchestrator_url: orchestrator_url.clone(),
|
||||||
max_turns: *max_turns,
|
max_turns: *max_turns,
|
||||||
model: model.clone(),
|
model: model.clone(),
|
||||||
timeout: std::time::Duration::from_secs(1800),
|
timeout: std::time::Duration::from_secs(1800),
|
||||||
allowed_tools: claude_config.allowed_tools,
|
allowed_tools: Vec::new(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let runtime = ironclaw::worker::ClaudeBridgeRuntime::new(config)
|
let runtime = ironclaw::worker::ClaudeBridgeRuntime::new(config)
|
||||||
@@ -235,15 +243,25 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
skip_auth,
|
skip_auth,
|
||||||
channels_only,
|
channels_only,
|
||||||
}) => {
|
}) => {
|
||||||
// Load .env before running onboarding wizard
|
// Load .env files before running onboarding wizard.
|
||||||
|
// Standard ./.env first (higher priority), then ~/.ironclaw/.env.
|
||||||
let _ = dotenvy::dotenv();
|
let _ = dotenvy::dotenv();
|
||||||
|
ironclaw::bootstrap::load_ironclaw_env();
|
||||||
|
|
||||||
let config = SetupConfig {
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
skip_auth: *skip_auth,
|
{
|
||||||
channels_only: *channels_only,
|
let config = SetupConfig {
|
||||||
};
|
skip_auth: *skip_auth,
|
||||||
let mut wizard = SetupWizard::with_config(config);
|
channels_only: *channels_only,
|
||||||
wizard.run().await?;
|
};
|
||||||
|
let mut wizard = SetupWizard::with_config(config);
|
||||||
|
wizard.run().await?;
|
||||||
|
}
|
||||||
|
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||||
|
{
|
||||||
|
let _ = (skip_auth, channels_only);
|
||||||
|
eprintln!("Onboarding wizard requires the 'postgres' or 'libsql' feature.");
|
||||||
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
None | Some(Command::Run) => {
|
None | Some(Command::Run) => {
|
||||||
@@ -251,22 +269,23 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load .env if present
|
// Load .env files early so DATABASE_URL (and any other vars) are
|
||||||
|
// available to all subsequent env-based config resolution.
|
||||||
|
// Standard ./.env first (higher priority), then ~/.ironclaw/.env.
|
||||||
let _ = dotenvy::dotenv();
|
let _ = dotenvy::dotenv();
|
||||||
|
ironclaw::bootstrap::load_ironclaw_env();
|
||||||
|
|
||||||
// Enhanced first-run detection
|
// Enhanced first-run detection
|
||||||
if !cli.no_onboard {
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
if let Some(reason) = check_onboard_needed().await {
|
if !cli.no_onboard
|
||||||
println!("Onboarding needed: {}", reason);
|
&& let Some(reason) = check_onboard_needed()
|
||||||
println!();
|
{
|
||||||
let mut wizard = SetupWizard::new();
|
println!("Onboarding needed: {}", reason);
|
||||||
wizard.run().await?;
|
println!();
|
||||||
}
|
let mut wizard = SetupWizard::new();
|
||||||
|
wizard.run().await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Load bootstrap config (4 fields that must live on disk)
|
|
||||||
let bootstrap = ironclaw::bootstrap::BootstrapConfig::load();
|
|
||||||
|
|
||||||
// Load initial config from env + disk (before DB is available)
|
// Load initial config from env + disk (before DB is available)
|
||||||
let mut config = match Config::from_env().await {
|
let mut config = match Config::from_env().await {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
@@ -286,7 +305,6 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
let session_config = SessionConfig {
|
let session_config = SessionConfig {
|
||||||
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
auth_base_url: config.llm.nearai.auth_base_url.clone(),
|
||||||
session_path: config.llm.nearai.session_path.clone(),
|
session_path: config.llm.nearai.session_path.clone(),
|
||||||
..Default::default()
|
|
||||||
};
|
};
|
||||||
let session = create_session_manager(session_config).await;
|
let session = create_session_manager(session_config).await;
|
||||||
|
|
||||||
@@ -297,7 +315,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// Initialize tracing
|
// Initialize tracing
|
||||||
let env_filter = EnvFilter::try_from_default_env()
|
let env_filter = EnvFilter::try_from_default_env()
|
||||||
.unwrap_or_else(|_| EnvFilter::new("ironclaw=info,tower_http=debug"));
|
.unwrap_or_else(|_| EnvFilter::new("ironclaw=info,tower_http=warn"));
|
||||||
|
|
||||||
// Create log broadcaster before tracing init so the WebLogLayer can capture all events.
|
// Create log broadcaster before tracing init so the WebLogLayer can capture all events.
|
||||||
// This gets wired to the gateway's /api/logs/events SSE endpoint later.
|
// This gets wired to the gateway's /api/logs/events SSE endpoint later.
|
||||||
@@ -305,7 +323,11 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
tracing_subscriber::registry()
|
tracing_subscriber::registry()
|
||||||
.with(env_filter)
|
.with(env_filter)
|
||||||
.with(tracing_subscriber::fmt::layer().with_target(false))
|
.with(
|
||||||
|
tracing_subscriber::fmt::layer()
|
||||||
|
.with_target(false)
|
||||||
|
.with_writer(ironclaw::tracing_fmt::TruncatingStderr::default()),
|
||||||
|
)
|
||||||
.with(WebLogLayer::new(Arc::clone(&log_broadcaster)))
|
.with(WebLogLayer::new(Arc::clone(&log_broadcaster)))
|
||||||
.init();
|
.init();
|
||||||
|
|
||||||
@@ -322,23 +344,86 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
tracing::info!("Loaded configuration for agent: {}", config.agent.name);
|
tracing::info!("Loaded configuration for agent: {}", config.agent.name);
|
||||||
tracing::info!("LLM backend: {}", config.llm.backend);
|
tracing::info!("LLM backend: {}", config.llm.backend);
|
||||||
|
|
||||||
// Initialize database store (optional for testing)
|
// Initialize database backend.
|
||||||
let store = if cli.no_db {
|
//
|
||||||
|
// Creates an `Arc<dyn Database>` that all consumers share.
|
||||||
|
// Backend is selected by the `DATABASE_BACKEND` env var / config.
|
||||||
|
//
|
||||||
|
// NOTE: For simpler call sites (CLI commands, Memory handler) use the shared
|
||||||
|
// helper `ironclaw::db::connect_from_config()`. This block is kept inline
|
||||||
|
// because it also captures backend-specific handles (`pg_pool`, `libsql_db`)
|
||||||
|
// needed by the secrets store.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
let mut pg_pool: Option<deadpool_postgres::Pool> = None;
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
let mut libsql_db: Option<std::sync::Arc<libsql::Database>> = None;
|
||||||
|
|
||||||
|
let db: Option<Arc<dyn ironclaw::db::Database>> = if cli.no_db {
|
||||||
tracing::warn!("Running without database connection");
|
tracing::warn!("Running without database connection");
|
||||||
None
|
None
|
||||||
} else {
|
} else {
|
||||||
let store = Store::new(&config.database).await?;
|
match config.database.backend {
|
||||||
store.run_migrations().await?;
|
#[cfg(feature = "libsql")]
|
||||||
tracing::info!("Database connected and migrations applied");
|
ironclaw::config::DatabaseBackend::LibSql => {
|
||||||
|
use ironclaw::db::Database as _;
|
||||||
|
use ironclaw::db::libsql_backend::LibSqlBackend;
|
||||||
|
use secrecy::ExposeSecret as _;
|
||||||
|
|
||||||
|
let default_path = ironclaw::config::default_libsql_path();
|
||||||
|
let db_path = config
|
||||||
|
.database
|
||||||
|
.libsql_path
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&default_path);
|
||||||
|
|
||||||
|
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||||
|
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||||
|
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||||
|
})?;
|
||||||
|
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret()).await?
|
||||||
|
} else {
|
||||||
|
LibSqlBackend::new_local(db_path).await?
|
||||||
|
};
|
||||||
|
backend.run_migrations().await?;
|
||||||
|
tracing::info!("libSQL database connected and migrations applied");
|
||||||
|
|
||||||
|
// Capture the Database handle for SecretsStore (connection-per-op)
|
||||||
|
libsql_db = Some(backend.shared_db());
|
||||||
|
|
||||||
|
Some(Arc::new(backend) as Arc<dyn ironclaw::db::Database>)
|
||||||
|
}
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
_ => {
|
||||||
|
use ironclaw::db::Database as _;
|
||||||
|
let pg = ironclaw::db::postgres::PgBackend::new(&config.database)
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
pg.run_migrations()
|
||||||
|
.await
|
||||||
|
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||||
|
tracing::info!("PostgreSQL database connected and migrations applied");
|
||||||
|
|
||||||
|
pg_pool = Some(pg.pool());
|
||||||
|
Some(Arc::new(pg) as Arc<dyn ironclaw::db::Database>)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "postgres"))]
|
||||||
|
_ => {
|
||||||
|
anyhow::bail!(
|
||||||
|
"No database backend available. Enable 'postgres' or 'libsql' feature."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Post-init operations using the database
|
||||||
|
if let Some(ref db) = db {
|
||||||
// One-time migration: move disk config files into the DB settings table.
|
// One-time migration: move disk config files into the DB settings table.
|
||||||
if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(&store, "default").await {
|
if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(db.as_ref(), "default").await {
|
||||||
tracing::warn!("Disk-to-DB settings migration failed: {}", e);
|
tracing::warn!("Disk-to-DB settings migration failed: {}", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reload config from DB now that we have a connection.
|
// Reload config from DB now that we have a connection.
|
||||||
// Priority: env var > DB setting > default.
|
match Config::from_db(db.as_ref(), "default").await {
|
||||||
match Config::from_db(&store, "default", &bootstrap).await {
|
|
||||||
Ok(db_config) => {
|
Ok(db_config) => {
|
||||||
config = db_config;
|
config = db_config;
|
||||||
tracing::info!("Configuration reloaded from database");
|
tracing::info!("Configuration reloaded from database");
|
||||||
@@ -351,23 +436,39 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let store = Arc::new(store);
|
// Attach DB to session manager so tokens save to DB too
|
||||||
|
session.attach_store(Arc::clone(db), "default").await;
|
||||||
// Attach store to session manager so tokens save to DB too
|
|
||||||
session.attach_store(Arc::clone(&store), "default").await;
|
|
||||||
|
|
||||||
// Mark any jobs left in "running" or "creating" state as "interrupted".
|
// Mark any jobs left in "running" or "creating" state as "interrupted".
|
||||||
if let Err(e) = store.cleanup_stale_sandbox_jobs().await {
|
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
|
||||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
Some(store)
|
|
||||||
};
|
|
||||||
|
|
||||||
// Initialize LLM provider (clone session so we can reuse it for embeddings)
|
// Initialize LLM provider (clone session so we can reuse it for embeddings)
|
||||||
let llm = create_llm_provider(&config.llm, session.clone())?;
|
let llm = create_llm_provider(&config.llm, session.clone())?;
|
||||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||||
|
|
||||||
|
// Wrap in failover if a fallback model is configured
|
||||||
|
let llm: Arc<dyn LlmProvider> =
|
||||||
|
if let Some(fallback_model) = config.llm.nearai.fallback_model.as_ref() {
|
||||||
|
if fallback_model == &config.llm.nearai.model {
|
||||||
|
tracing::warn!(
|
||||||
|
"fallback_model is the same as primary model, failover may not be effective"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut fallback_config = config.llm.nearai.clone();
|
||||||
|
fallback_config.model = fallback_model.clone();
|
||||||
|
let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?;
|
||||||
|
tracing::info!(
|
||||||
|
primary = %llm.model_name(),
|
||||||
|
fallback = %fallback.model_name(),
|
||||||
|
"LLM failover enabled"
|
||||||
|
);
|
||||||
|
Arc::new(FailoverProvider::new(vec![llm, fallback])?)
|
||||||
|
} else {
|
||||||
|
llm
|
||||||
|
};
|
||||||
|
|
||||||
// Initialize safety layer
|
// Initialize safety layer
|
||||||
let safety = Arc::new(SafetyLayer::new(&config.safety));
|
let safety = Arc::new(SafetyLayer::new(&config.safety));
|
||||||
tracing::info!("Safety layer initialized");
|
tracing::info!("Safety layer initialized");
|
||||||
@@ -417,8 +518,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Register memory tools if database is available
|
// Register memory tools if database is available
|
||||||
if let Some(ref store) = store {
|
if let Some(ref db) = db {
|
||||||
let mut workspace = Workspace::new("default", store.pool());
|
let mut workspace = Workspace::new_with_db("default", Arc::clone(db));
|
||||||
if let Some(ref emb) = embeddings {
|
if let Some(ref emb) = embeddings {
|
||||||
workspace = workspace.with_embeddings(emb.clone());
|
workspace = workspace.with_embeddings(emb.clone());
|
||||||
}
|
}
|
||||||
@@ -441,20 +542,46 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
tracing::info!("Builder mode enabled");
|
tracing::info!("Builder mode enabled");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create secrets store if master key is configured (needed for MCP auth and WASM channels)
|
// Create secrets store if master key is configured (needed for MCP auth and WASM channels).
|
||||||
|
//
|
||||||
|
// When both `postgres` and `libsql` features are compiled, the runtime-selected
|
||||||
|
// backend determines which store is created: whichever DB init branch ran will
|
||||||
|
// have set its handle (pg_pool or libsql_db), and the or_else chain picks it up.
|
||||||
let secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>> =
|
let secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>> =
|
||||||
if let (Some(store), Some(master_key)) = (&store, config.secrets.master_key()) {
|
if let Some(master_key) = config.secrets.master_key() {
|
||||||
match SecretsCrypto::new(master_key.clone()) {
|
match SecretsCrypto::new(master_key.clone()) {
|
||||||
Ok(crypto) => Some(Arc::new(PostgresSecretsStore::new(
|
Ok(crypto) => {
|
||||||
store.pool(),
|
let crypto = Arc::new(crypto);
|
||||||
Arc::new(crypto),
|
let store: Option<Arc<dyn SecretsStore + Send + Sync>> = None;
|
||||||
))),
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
let store = store.or_else(|| {
|
||||||
|
libsql_db.take().map(|db| {
|
||||||
|
Arc::new(LibSqlSecretsStore::new(db, Arc::clone(&crypto)))
|
||||||
|
as Arc<dyn SecretsStore + Send + Sync>
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
let store = store.or_else(|| {
|
||||||
|
pg_pool.as_ref().map(|pool| {
|
||||||
|
Arc::new(PostgresSecretsStore::new(pool.clone(), Arc::clone(&crypto)))
|
||||||
|
as Arc<dyn SecretsStore + Send + Sync>
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
store
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Failed to initialize secrets crypto: {}", e);
|
tracing::warn!("Failed to initialize secrets crypto: {}", e);
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
let _ = libsql_db.take();
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
let _ = libsql_db.take();
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -478,7 +605,10 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
// Both register into the shared ToolRegistry (RwLock-based) so concurrent writes are safe.
|
// Both register into the shared ToolRegistry (RwLock-based) so concurrent writes are safe.
|
||||||
let wasm_tools_future = async {
|
let wasm_tools_future = async {
|
||||||
if let Some(ref runtime) = wasm_tool_runtime {
|
if let Some(ref runtime) = wasm_tool_runtime {
|
||||||
let loader = WasmToolLoader::new(Arc::clone(runtime), Arc::clone(&tools));
|
let mut loader = WasmToolLoader::new(Arc::clone(runtime), Arc::clone(&tools));
|
||||||
|
if let Some(ref secrets) = secrets_store {
|
||||||
|
loader = loader.with_secrets_store(Arc::clone(secrets));
|
||||||
|
}
|
||||||
|
|
||||||
// Load installed tools from ~/.ironclaw/tools/
|
// Load installed tools from ~/.ironclaw/tools/
|
||||||
match loader.load_from_dir(&config.wasm.tools_dir).await {
|
match loader.load_from_dir(&config.wasm.tools_dir).await {
|
||||||
@@ -518,8 +648,8 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
let mcp_servers_future = async {
|
let mcp_servers_future = async {
|
||||||
if let Some(ref secrets) = secrets_store {
|
if let Some(ref secrets) = secrets_store {
|
||||||
let servers_result = if let Some(ref s) = store {
|
let servers_result = if let Some(ref d) = db {
|
||||||
load_mcp_servers_from_db(s, "default").await
|
load_mcp_servers_from_db(d.as_ref(), "default").await
|
||||||
} else {
|
} else {
|
||||||
ironclaw::tools::mcp::config::load_mcp_servers().await
|
ironclaw::tools::mcp::config::load_mcp_servers().await
|
||||||
};
|
};
|
||||||
@@ -632,7 +762,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
config.channels.wasm_channels_dir.clone(),
|
config.channels.wasm_channels_dir.clone(),
|
||||||
config.tunnel.public_url.clone(),
|
config.tunnel.public_url.clone(),
|
||||||
"default".to_string(),
|
"default".to_string(),
|
||||||
store.clone(),
|
db.clone(),
|
||||||
));
|
));
|
||||||
tools.register_extension_tools(Arc::clone(&manager));
|
tools.register_extension_tools(Arc::clone(&manager));
|
||||||
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
||||||
@@ -695,7 +825,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
token_store,
|
token_store,
|
||||||
job_event_tx: job_event_tx.clone(),
|
job_event_tx: job_event_tx.clone(),
|
||||||
prompt_queue: Arc::clone(&prompt_queue),
|
prompt_queue: Arc::clone(&prompt_queue),
|
||||||
store: store.clone(),
|
store: db.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -801,13 +931,13 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// Inject owner_id for Telegram so the bot only responds
|
// Inject owner_id for Telegram so the bot only responds
|
||||||
// to the bound user account.
|
// to the bound user account.
|
||||||
if channel_name == "telegram" {
|
if channel_name == "telegram"
|
||||||
if let Some(owner_id) = config.channels.telegram_owner_id {
|
&& let Some(owner_id) = config.channels.telegram_owner_id
|
||||||
config_updates.insert(
|
{
|
||||||
"owner_id".to_string(),
|
config_updates.insert(
|
||||||
serde_json::json!(owner_id),
|
"owner_id".to_string(),
|
||||||
);
|
serde_json::json!(owner_id),
|
||||||
}
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config_updates.is_empty() {
|
if !config_updates.is_empty() {
|
||||||
@@ -898,23 +1028,23 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
// Extract its routes for the unified server; the channel itself just
|
// Extract its routes for the unified server; the channel itself just
|
||||||
// provides the mpsc stream.
|
// provides the mpsc stream.
|
||||||
let mut webhook_server_addr: Option<std::net::SocketAddr> = None;
|
let mut webhook_server_addr: Option<std::net::SocketAddr> = None;
|
||||||
if !cli.cli_only {
|
if !cli.cli_only
|
||||||
if let Some(ref http_config) = config.channels.http {
|
&& let Some(ref http_config) = config.channels.http
|
||||||
let http_channel = HttpChannel::new(http_config.clone());
|
{
|
||||||
webhook_routes.push(http_channel.routes());
|
let http_channel = HttpChannel::new(http_config.clone());
|
||||||
let (host, port) = http_channel.addr();
|
webhook_routes.push(http_channel.routes());
|
||||||
webhook_server_addr = Some(
|
let (host, port) = http_channel.addr();
|
||||||
format!("{}:{}", host, port)
|
webhook_server_addr = Some(
|
||||||
.parse()
|
format!("{}:{}", host, port)
|
||||||
.expect("HttpConfig host:port must be a valid SocketAddr"),
|
.parse()
|
||||||
);
|
.expect("HttpConfig host:port must be a valid SocketAddr"),
|
||||||
channels.add(Box::new(http_channel));
|
);
|
||||||
tracing::info!(
|
channels.add(Box::new(http_channel));
|
||||||
"HTTP channel enabled on {}:{}",
|
tracing::info!(
|
||||||
http_config.host,
|
"HTTP channel enabled on {}:{}",
|
||||||
http_config.port
|
http_config.host,
|
||||||
);
|
http_config.port
|
||||||
}
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start the unified webhook server if any routes were registered.
|
// Start the unified webhook server if any routes were registered.
|
||||||
@@ -932,13 +1062,15 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Create workspace for agent (shared with memory tools)
|
// Create workspace for agent (shared with memory tools)
|
||||||
let workspace = store.as_ref().map(|s| {
|
let workspace = if let Some(ref db_ref) = db {
|
||||||
let mut ws = Workspace::new("default", s.pool());
|
let mut ws = Workspace::new_with_db("default", Arc::clone(db_ref));
|
||||||
if let Some(ref emb) = embeddings {
|
if let Some(ref emb) = embeddings {
|
||||||
ws = ws.with_embeddings(emb.clone());
|
ws = ws.with_embeddings(emb.clone());
|
||||||
}
|
}
|
||||||
Arc::new(ws)
|
Some(Arc::new(ws))
|
||||||
});
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
// Seed workspace with core identity files on first boot
|
// Seed workspace with core identity files on first boot
|
||||||
if let Some(ref ws) = workspace {
|
if let Some(ref ws) = workspace {
|
||||||
@@ -976,7 +1108,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
tools.register_job_tools(
|
tools.register_job_tools(
|
||||||
Arc::clone(&context_manager),
|
Arc::clone(&context_manager),
|
||||||
container_job_manager.clone(),
|
container_job_manager.clone(),
|
||||||
store.clone(),
|
db.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Add web gateway channel if configured
|
// Add web gateway channel if configured
|
||||||
@@ -991,13 +1123,12 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
if let Some(ref ext_mgr) = extension_manager {
|
if let Some(ref ext_mgr) = extension_manager {
|
||||||
gw = gw.with_extension_manager(Arc::clone(ext_mgr));
|
gw = gw.with_extension_manager(Arc::clone(ext_mgr));
|
||||||
}
|
}
|
||||||
if let Some(ref s) = store {
|
if let Some(ref d) = db {
|
||||||
gw = gw.with_store(Arc::clone(s));
|
gw = gw.with_store(Arc::clone(d));
|
||||||
}
|
}
|
||||||
if let Some(ref jm) = container_job_manager {
|
if let Some(ref jm) = container_job_manager {
|
||||||
gw = gw.with_job_manager(Arc::clone(jm));
|
gw = gw.with_job_manager(Arc::clone(jm));
|
||||||
}
|
}
|
||||||
gw = gw.with_llm_provider(Arc::clone(&llm));
|
|
||||||
if config.sandbox.enabled {
|
if config.sandbox.enabled {
|
||||||
gw = gw.with_prompt_queue(Arc::clone(&prompt_queue));
|
gw = gw.with_prompt_queue(Arc::clone(&prompt_queue));
|
||||||
|
|
||||||
@@ -1030,7 +1161,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
// Create and run the agent
|
// Create and run the agent
|
||||||
let deps = AgentDeps {
|
let deps = AgentDeps {
|
||||||
store,
|
store: db,
|
||||||
llm,
|
llm,
|
||||||
safety,
|
safety,
|
||||||
tools,
|
tools,
|
||||||
@@ -1064,29 +1195,18 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
/// Check if onboarding is needed and return the reason.
|
/// Check if onboarding is needed and return the reason.
|
||||||
///
|
///
|
||||||
/// Returns `Some(reason)` if onboarding should be triggered, `None` otherwise.
|
/// Returns `Some(reason)` if onboarding should be triggered, `None` otherwise.
|
||||||
async fn check_onboard_needed() -> Option<&'static str> {
|
/// Called after `load_ironclaw_env()`, so DATABASE_URL from `~/.ironclaw/.env`
|
||||||
let bootstrap = ironclaw::bootstrap::BootstrapConfig::load();
|
/// is already in the environment.
|
||||||
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
|
fn check_onboard_needed() -> Option<&'static str> {
|
||||||
|
let has_db = std::env::var("DATABASE_URL").is_ok()
|
||||||
|
|| std::env::var("LIBSQL_PATH").is_ok()
|
||||||
|
|| ironclaw::config::default_libsql_path().exists();
|
||||||
|
|
||||||
// Database not configured (and not in env)
|
if !has_db {
|
||||||
if bootstrap.database_url.is_none() && std::env::var("DATABASE_URL").is_err() {
|
|
||||||
return Some("Database not configured");
|
return Some("Database not configured");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Secrets not configured (and not in env)
|
|
||||||
if bootstrap.secrets_master_key_source == ironclaw::settings::KeySource::None
|
|
||||||
&& std::env::var("SECRETS_MASTER_KEY").is_err()
|
|
||||||
&& !ironclaw::secrets::keychain::has_master_key().await
|
|
||||||
{
|
|
||||||
// Only require secrets setup if user hasn't explicitly disabled it
|
|
||||||
// For now, we don't require it for first run
|
|
||||||
}
|
|
||||||
|
|
||||||
// First run (onboarding never completed and no session)
|
|
||||||
let session_path = ironclaw::llm::session::default_session_path();
|
|
||||||
if !bootstrap.onboard_completed && !session_path.exists() {
|
|
||||||
return Some("First run");
|
|
||||||
}
|
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+14
-14
@@ -15,7 +15,7 @@ use tokio::sync::{Mutex, broadcast};
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::channels::web::types::SseEvent;
|
use crate::channels::web::types::SseEvent;
|
||||||
use crate::history::Store;
|
use crate::db::Database;
|
||||||
use crate::llm::{CompletionRequest, LlmProvider, ToolCompletionRequest};
|
use crate::llm::{CompletionRequest, LlmProvider, ToolCompletionRequest};
|
||||||
use crate::orchestrator::auth::{TokenStore, worker_auth_middleware};
|
use crate::orchestrator::auth::{TokenStore, worker_auth_middleware};
|
||||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||||
@@ -43,7 +43,7 @@ pub struct OrchestratorState {
|
|||||||
/// Buffered follow-up prompts for sandbox jobs, keyed by job_id.
|
/// Buffered follow-up prompts for sandbox jobs, keyed by job_id.
|
||||||
pub prompt_queue: Arc<Mutex<HashMap<Uuid, VecDeque<PendingPrompt>>>>,
|
pub prompt_queue: Arc<Mutex<HashMap<Uuid, VecDeque<PendingPrompt>>>>,
|
||||||
/// Database handle for persisting job events.
|
/// Database handle for persisting job events.
|
||||||
pub store: Option<Arc<Store>>,
|
pub store: Option<Arc<dyn Database>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The orchestrator's internal API server.
|
/// The orchestrator's internal API server.
|
||||||
@@ -202,7 +202,7 @@ async fn report_complete(
|
|||||||
State(state): State<OrchestratorState>,
|
State(state): State<OrchestratorState>,
|
||||||
Path(job_id): Path<Uuid>,
|
Path(job_id): Path<Uuid>,
|
||||||
Json(report): Json<CompletionReport>,
|
Json(report): Json<CompletionReport>,
|
||||||
) -> Result<StatusCode, StatusCode> {
|
) -> Result<Json<serde_json::Value>, StatusCode> {
|
||||||
if report.success {
|
if report.success {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
job_id = %job_id,
|
job_id = %job_id,
|
||||||
@@ -223,7 +223,7 @@ async fn report_complete(
|
|||||||
};
|
};
|
||||||
let _ = state.job_manager.complete_job(job_id, result).await;
|
let _ = state.job_manager.complete_job(job_id, result).await;
|
||||||
|
|
||||||
Ok(StatusCode::OK)
|
Ok(Json(serde_json::json!({"status": "ok"})))
|
||||||
}
|
}
|
||||||
|
|
||||||
// -- Sandbox job event handlers --
|
// -- Sandbox job event handlers --
|
||||||
@@ -339,16 +339,16 @@ async fn get_prompt_handler(
|
|||||||
Path(job_id): Path<Uuid>,
|
Path(job_id): Path<Uuid>,
|
||||||
) -> Result<(StatusCode, Json<serde_json::Value>), StatusCode> {
|
) -> Result<(StatusCode, Json<serde_json::Value>), StatusCode> {
|
||||||
let mut queue = state.prompt_queue.lock().await;
|
let mut queue = state.prompt_queue.lock().await;
|
||||||
if let Some(prompts) = queue.get_mut(&job_id) {
|
if let Some(prompts) = queue.get_mut(&job_id)
|
||||||
if let Some(prompt) = prompts.pop_front() {
|
&& let Some(prompt) = prompts.pop_front()
|
||||||
return Ok((
|
{
|
||||||
StatusCode::OK,
|
return Ok((
|
||||||
Json(serde_json::json!({
|
StatusCode::OK,
|
||||||
"content": prompt.content,
|
Json(serde_json::json!({
|
||||||
"done": prompt.done,
|
"content": prompt.content,
|
||||||
})),
|
"done": prompt.done,
|
||||||
));
|
})),
|
||||||
}
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return 204 with an empty body. The Json wrapper requires some value
|
// Return 204 with an empty body. The Json wrapper requires some value
|
||||||
|
|||||||
@@ -229,17 +229,17 @@ impl ContainerJobManager {
|
|||||||
.unwrap_or_else(|| PathBuf::from("."))
|
.unwrap_or_else(|| PathBuf::from("."))
|
||||||
.join(".ironclaw")
|
.join(".ironclaw")
|
||||||
.join("projects");
|
.join("projects");
|
||||||
if let Ok(canonical_base) = projects_base.canonicalize() {
|
if let Ok(canonical_base) = projects_base.canonicalize()
|
||||||
if !canonical.starts_with(&canonical_base) {
|
&& !canonical.starts_with(&canonical_base)
|
||||||
return Err(OrchestratorError::ContainerCreationFailed {
|
{
|
||||||
job_id,
|
return Err(OrchestratorError::ContainerCreationFailed {
|
||||||
reason: format!(
|
job_id,
|
||||||
"project directory {} is outside allowed base {}",
|
reason: format!(
|
||||||
canonical.display(),
|
"project directory {} is outside allowed base {}",
|
||||||
canonical_base.display()
|
canonical.display(),
|
||||||
),
|
canonical_base.display()
|
||||||
});
|
),
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
binds.push(format!("{}:/workspace:rw", canonical.display()));
|
binds.push(format!("{}:/workspace:rw", canonical.display()));
|
||||||
env_vec.push("IRONCLAW_WORKSPACE=/workspace".to_string());
|
env_vec.push("IRONCLAW_WORKSPACE=/workspace".to_string());
|
||||||
@@ -442,36 +442,36 @@ impl ContainerJobManager {
|
|||||||
let containers = self.containers.read().await;
|
let containers = self.containers.read().await;
|
||||||
containers.get(&job_id).map(|h| h.container_id.clone())
|
containers.get(&job_id).map(|h| h.container_id.clone())
|
||||||
};
|
};
|
||||||
if let Some(cid) = container_id {
|
if let Some(cid) = container_id
|
||||||
if !cid.is_empty() {
|
&& !cid.is_empty()
|
||||||
match connect_docker().await {
|
{
|
||||||
Ok(docker) => {
|
match connect_docker().await {
|
||||||
if let Err(e) = docker
|
Ok(docker) => {
|
||||||
.stop_container(
|
if let Err(e) = docker
|
||||||
&cid,
|
.stop_container(
|
||||||
Some(bollard::container::StopContainerOptions { t: 5 }),
|
&cid,
|
||||||
)
|
Some(bollard::container::StopContainerOptions { t: 5 }),
|
||||||
.await
|
)
|
||||||
{
|
.await
|
||||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop completed container");
|
{
|
||||||
}
|
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop completed container");
|
||||||
if let Err(e) = docker
|
|
||||||
.remove_container(
|
|
||||||
&cid,
|
|
||||||
Some(bollard::container::RemoveContainerOptions {
|
|
||||||
force: true,
|
|
||||||
..Default::default()
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to remove completed container");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
Err(e) => {
|
if let Err(e) = docker
|
||||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to connect to Docker for container cleanup");
|
.remove_container(
|
||||||
|
&cid,
|
||||||
|
Some(bollard::container::RemoveContainerOptions {
|
||||||
|
force: true,
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!(job_id = %job_id, error = %e, "Failed to remove completed container");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!(job_id = %job_id, error = %e, "Failed to connect to Docker for container cleanup");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.token_store.revoke(job_id).await;
|
self.token_store.revoke(job_id).await;
|
||||||
|
|||||||
@@ -147,10 +147,10 @@ impl LeakDetector {
|
|||||||
// Build prefix matcher for patterns that start with a known prefix
|
// Build prefix matcher for patterns that start with a known prefix
|
||||||
let mut prefixes = Vec::new();
|
let mut prefixes = Vec::new();
|
||||||
for (idx, pattern) in patterns.iter().enumerate() {
|
for (idx, pattern) in patterns.iter().enumerate() {
|
||||||
if let Some(prefix) = extract_literal_prefix(pattern.regex.as_str()) {
|
if let Some(prefix) = extract_literal_prefix(pattern.regex.as_str())
|
||||||
if prefix.len() >= 3 {
|
&& prefix.len() >= 3
|
||||||
prefixes.push((prefix, idx));
|
{
|
||||||
}
|
prefixes.push((prefix, idx));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -494,10 +494,10 @@ impl ContainerRunner {
|
|||||||
/// 3. `~/.docker/run/docker.sock` (Docker Desktop on macOS)
|
/// 3. `~/.docker/run/docker.sock` (Docker Desktop on macOS)
|
||||||
pub async fn connect_docker() -> Result<Docker> {
|
pub async fn connect_docker() -> Result<Docker> {
|
||||||
// First try bollard defaults (checks DOCKER_HOST, then /var/run/docker.sock)
|
// First try bollard defaults (checks DOCKER_HOST, then /var/run/docker.sock)
|
||||||
if let Ok(docker) = Docker::connect_with_local_defaults() {
|
if let Ok(docker) = Docker::connect_with_local_defaults()
|
||||||
if docker.ping().await.is_ok() {
|
&& docker.ping().await.is_ok()
|
||||||
return Ok(docker);
|
{
|
||||||
}
|
return Ok(docker);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try Docker Desktop socket (macOS)
|
// Try Docker Desktop socket (macOS)
|
||||||
@@ -507,10 +507,9 @@ pub async fn connect_docker() -> Result<Docker> {
|
|||||||
let sock_str = desktop_sock.to_string_lossy();
|
let sock_str = desktop_sock.to_string_lossy();
|
||||||
if let Ok(docker) =
|
if let Ok(docker) =
|
||||||
Docker::connect_with_socket(&sock_str, 120, bollard::API_DEFAULT_VERSION)
|
Docker::connect_with_socket(&sock_str, 120, bollard::API_DEFAULT_VERSION)
|
||||||
|
&& docker.ping().await.is_ok()
|
||||||
{
|
{
|
||||||
if docker.ping().await.is_ok() {
|
return Ok(docker);
|
||||||
return Ok(docker);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -259,11 +259,11 @@ async fn handle_connect(
|
|||||||
|
|
||||||
let decision = state.decider.decide(&network_req).await;
|
let decision = state.decider.decide(&network_req).await;
|
||||||
|
|
||||||
if !decision.is_allowed() {
|
if !decision.is_allowed()
|
||||||
if let NetworkDecision::Deny { reason } = decision {
|
&& let NetworkDecision::Deny { reason } = decision
|
||||||
tracing::info!("Proxy: blocked CONNECT {} - {}", host, reason);
|
{
|
||||||
return error_response(StatusCode::FORBIDDEN, reason);
|
tracing::info!("Proxy: blocked CONNECT {} - {}", host, reason);
|
||||||
}
|
return error_response(StatusCode::FORBIDDEN, reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
tracing::debug!("Proxy: allowing CONNECT to {}", host);
|
tracing::debug!("Proxy: allowing CONNECT to {}", host);
|
||||||
@@ -294,10 +294,10 @@ async fn forward_request(
|
|||||||
|
|
||||||
// Copy headers (except hop-by-hop headers)
|
// Copy headers (except hop-by-hop headers)
|
||||||
for (name, value) in req.headers() {
|
for (name, value) in req.headers() {
|
||||||
if !is_hop_by_hop_header(name.as_str()) {
|
if !is_hop_by_hop_header(name.as_str())
|
||||||
if let Ok(v) = value.to_str() {
|
&& let Ok(v) = value.to_str()
|
||||||
builder = builder.header(name.as_str(), v);
|
{
|
||||||
}
|
builder = builder.header(name.as_str(), v);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -109,12 +109,11 @@ impl NetworkPolicyDecider for DefaultPolicyDecider {
|
|||||||
async fn decide(&self, request: &NetworkRequest) -> NetworkDecision {
|
async fn decide(&self, request: &NetworkRequest) -> NetworkDecision {
|
||||||
// First check if the domain is allowed
|
// First check if the domain is allowed
|
||||||
let validation = self.allowlist.is_allowed(&request.host);
|
let validation = self.allowlist.is_allowed(&request.host);
|
||||||
if !validation.is_allowed() {
|
if !validation.is_allowed()
|
||||||
if let crate::sandbox::proxy::allowlist::DomainValidationResult::Denied(reason) =
|
&& let crate::sandbox::proxy::allowlist::DomainValidationResult::Denied(reason) =
|
||||||
validation
|
validation
|
||||||
{
|
{
|
||||||
return NetworkDecision::Deny { reason };
|
return NetworkDecision::Deny { reason };
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if we need to inject credentials
|
// Check if we need to inject credentials
|
||||||
|
|||||||
@@ -261,7 +261,7 @@ pub use platform::{delete_master_key, get_master_key, has_master_key, store_mast
|
|||||||
|
|
||||||
/// Parse a hex string to bytes.
|
/// Parse a hex string to bytes.
|
||||||
fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, SecretError> {
|
fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, SecretError> {
|
||||||
if hex.len() % 2 != 0 {
|
if !hex.len().is_multiple_of(2) {
|
||||||
return Err(SecretError::KeychainError(
|
return Err(SecretError::KeychainError(
|
||||||
"Invalid hex string length".to_string(),
|
"Invalid hex string length".to_string(),
|
||||||
));
|
));
|
||||||
|
|||||||
+5
-1
@@ -64,7 +64,11 @@ mod store;
|
|||||||
mod types;
|
mod types;
|
||||||
|
|
||||||
pub use crypto::SecretsCrypto;
|
pub use crypto::SecretsCrypto;
|
||||||
pub use store::{PostgresSecretsStore, SecretsStore};
|
#[cfg(feature = "libsql")]
|
||||||
|
pub use store::LibSqlSecretsStore;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
pub use store::PostgresSecretsStore;
|
||||||
|
pub use store::SecretsStore;
|
||||||
pub use types::{
|
pub use types::{
|
||||||
CreateSecretParams, CredentialLocation, CredentialMapping, DecryptedSecret, Secret,
|
CreateSecretParams, CredentialLocation, CredentialMapping, DecryptedSecret, Secret,
|
||||||
SecretError, SecretRef,
|
SecretError, SecretRef,
|
||||||
|
|||||||
+382
-14
@@ -10,6 +10,7 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use deadpool_postgres::Pool;
|
use deadpool_postgres::Pool;
|
||||||
use secrecy::ExposeSecret;
|
use secrecy::ExposeSecret;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
@@ -61,11 +62,13 @@ pub trait SecretsStore: Send + Sync {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// PostgreSQL implementation of SecretsStore.
|
/// PostgreSQL implementation of SecretsStore.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub struct PostgresSecretsStore {
|
pub struct PostgresSecretsStore {
|
||||||
pool: Pool,
|
pool: Pool,
|
||||||
crypto: Arc<SecretsCrypto>,
|
crypto: Arc<SecretsCrypto>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl PostgresSecretsStore {
|
impl PostgresSecretsStore {
|
||||||
/// Create a new store with the given database pool and crypto instance.
|
/// Create a new store with the given database pool and crypto instance.
|
||||||
pub fn new(pool: Pool, crypto: Arc<SecretsCrypto>) -> Self {
|
pub fn new(pool: Pool, crypto: Arc<SecretsCrypto>) -> Self {
|
||||||
@@ -73,6 +76,7 @@ impl PostgresSecretsStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl SecretsStore for PostgresSecretsStore {
|
impl SecretsStore for PostgresSecretsStore {
|
||||||
async fn create(
|
async fn create(
|
||||||
@@ -149,10 +153,10 @@ impl SecretsStore for PostgresSecretsStore {
|
|||||||
let secret = row_to_secret(&r);
|
let secret = row_to_secret(&r);
|
||||||
|
|
||||||
// Check expiration
|
// Check expiration
|
||||||
if let Some(expires_at) = secret.expires_at {
|
if let Some(expires_at) = secret.expires_at
|
||||||
if expires_at < Utc::now() {
|
&& expires_at < Utc::now()
|
||||||
return Err(SecretError::Expired);
|
{
|
||||||
}
|
return Err(SecretError::Expired);
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(secret)
|
Ok(secret)
|
||||||
@@ -272,10 +276,10 @@ impl SecretsStore for PostgresSecretsStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Simple glob: * matches any suffix
|
// Simple glob: * matches any suffix
|
||||||
if let Some(prefix) = pattern.strip_suffix('*') {
|
if let Some(prefix) = pattern.strip_suffix('*')
|
||||||
if secret_name.starts_with(prefix) {
|
&& secret_name.starts_with(prefix)
|
||||||
return Ok(true);
|
{
|
||||||
}
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -283,6 +287,7 @@ impl SecretsStore for PostgresSecretsStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn row_to_secret(row: &tokio_postgres::Row) -> Secret {
|
fn row_to_secret(row: &tokio_postgres::Row) -> Secret {
|
||||||
Secret {
|
Secret {
|
||||||
id: row.get("id"),
|
id: row.get("id"),
|
||||||
@@ -299,6 +304,332 @@ fn row_to_secret(row: &tokio_postgres::Row) -> Secret {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== libSQL implementation ====================
|
||||||
|
|
||||||
|
/// libSQL/Turso implementation of SecretsStore.
|
||||||
|
///
|
||||||
|
/// Holds an `Arc<Database>` handle and creates a fresh connection per operation,
|
||||||
|
/// matching the connection-per-request pattern used by the main `LibSqlBackend`.
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
pub struct LibSqlSecretsStore {
|
||||||
|
db: Arc<libsql::Database>,
|
||||||
|
crypto: Arc<SecretsCrypto>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
impl LibSqlSecretsStore {
|
||||||
|
/// Create a new store with the given shared libsql database handle and crypto instance.
|
||||||
|
pub fn new(db: Arc<libsql::Database>, crypto: Arc<SecretsCrypto>) -> Self {
|
||||||
|
Self { db, crypto }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn connect(&self) -> Result<libsql::Connection, SecretError> {
|
||||||
|
self.db
|
||||||
|
.connect()
|
||||||
|
.map_err(|e| SecretError::Database(format!("Connection failed: {}", e)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
#[async_trait]
|
||||||
|
impl SecretsStore for LibSqlSecretsStore {
|
||||||
|
async fn create(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
params: CreateSecretParams,
|
||||||
|
) -> Result<Secret, SecretError> {
|
||||||
|
let plaintext = params.value.expose_secret().as_bytes();
|
||||||
|
let (encrypted_value, key_salt) = self.crypto.encrypt(plaintext)?;
|
||||||
|
|
||||||
|
let id = Uuid::new_v4();
|
||||||
|
let now = Utc::now();
|
||||||
|
let now_str = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||||
|
let expires_at_str = params
|
||||||
|
.expires_at
|
||||||
|
.map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
|
||||||
|
|
||||||
|
// Start transaction for atomic upsert + read-back
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let tx = conn
|
||||||
|
.transaction()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
tx.execute(
|
||||||
|
r#"
|
||||||
|
INSERT INTO secrets (id, user_id, name, encrypted_value, key_salt, provider, expires_at, created_at, updated_at)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8)
|
||||||
|
ON CONFLICT (user_id, name) DO UPDATE SET
|
||||||
|
encrypted_value = excluded.encrypted_value,
|
||||||
|
key_salt = excluded.key_salt,
|
||||||
|
provider = excluded.provider,
|
||||||
|
expires_at = excluded.expires_at,
|
||||||
|
updated_at = ?8
|
||||||
|
"#,
|
||||||
|
libsql::params![
|
||||||
|
id.to_string(),
|
||||||
|
user_id,
|
||||||
|
params.name.as_str(),
|
||||||
|
libsql::Value::Blob(encrypted_value.clone()),
|
||||||
|
libsql::Value::Blob(key_salt.clone()),
|
||||||
|
libsql_opt_text(params.provider.as_deref()),
|
||||||
|
libsql_opt_text(expires_at_str.as_deref()),
|
||||||
|
now_str.as_str(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
// Read back the row (may have been upserted)
|
||||||
|
let mut rows = tx
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at,
|
||||||
|
last_used_at, usage_count, created_at, updated_at
|
||||||
|
FROM secrets
|
||||||
|
WHERE user_id = ?1 AND name = ?2
|
||||||
|
"#,
|
||||||
|
libsql::params![user_id, params.name.as_str()],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
let row = rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||||
|
.ok_or_else(|| SecretError::Database("Insert succeeded but row not found".into()))?;
|
||||||
|
|
||||||
|
let secret = libsql_row_to_secret(&row)?;
|
||||||
|
|
||||||
|
tx.commit()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(secret)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(&self, user_id: &str, name: &str) -> Result<Secret, SecretError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at,
|
||||||
|
last_used_at, usage_count, created_at, updated_at
|
||||||
|
FROM secrets
|
||||||
|
WHERE user_id = ?1 AND name = ?2
|
||||||
|
"#,
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
match rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
Some(row) => {
|
||||||
|
let secret = libsql_row_to_secret(&row)?;
|
||||||
|
|
||||||
|
if let Some(expires_at) = secret.expires_at
|
||||||
|
&& expires_at < Utc::now()
|
||||||
|
{
|
||||||
|
return Err(SecretError::Expired);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(secret)
|
||||||
|
}
|
||||||
|
None => Err(SecretError::NotFound(name.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_decrypted(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<DecryptedSecret, SecretError> {
|
||||||
|
let secret = self.get(user_id, name).await?;
|
||||||
|
self.crypto
|
||||||
|
.decrypt(&secret.encrypted_value, &secret.key_salt)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn exists(&self, user_id: &str, name: &str) -> Result<bool, SecretError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
"SELECT 1 FROM secrets WHERE user_id = ?1 AND name = ?2",
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||||
|
.is_some())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(&self, user_id: &str) -> Result<Vec<SecretRef>, SecretError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
"SELECT name, provider FROM secrets WHERE user_id = ?1 ORDER BY name",
|
||||||
|
libsql::params![user_id],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
let mut refs = Vec::new();
|
||||||
|
while let Some(row) = rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
refs.push(SecretRef {
|
||||||
|
name: row.get::<String>(0).unwrap_or_default(),
|
||||||
|
provider: row.get::<String>(1).ok(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(refs)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, user_id: &str, name: &str) -> Result<bool, SecretError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let affected = conn
|
||||||
|
.execute(
|
||||||
|
"DELETE FROM secrets WHERE user_id = ?1 AND name = ?2",
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(affected > 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn record_usage(&self, secret_id: Uuid) -> Result<(), SecretError> {
|
||||||
|
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||||
|
let conn = self.connect()?;
|
||||||
|
|
||||||
|
conn.execute(
|
||||||
|
r#"
|
||||||
|
UPDATE secrets
|
||||||
|
SET last_used_at = ?1, usage_count = usage_count + 1
|
||||||
|
WHERE id = ?2
|
||||||
|
"#,
|
||||||
|
libsql::params![now.as_str(), secret_id.to_string()],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn is_accessible(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
secret_name: &str,
|
||||||
|
allowed_secrets: &[String],
|
||||||
|
) -> Result<bool, SecretError> {
|
||||||
|
if !self.exists(user_id, secret_name).await? {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
for pattern in allowed_secrets {
|
||||||
|
if pattern == secret_name {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(prefix) = pattern.strip_suffix('*')
|
||||||
|
&& secret_name.starts_with(prefix)
|
||||||
|
{
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_opt_text(s: Option<&str>) -> libsql::Value {
|
||||||
|
match s {
|
||||||
|
Some(s) => libsql::Value::Text(s.to_string()),
|
||||||
|
None => libsql::Value::Null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_parse_timestamp(s: &str) -> Result<chrono::DateTime<Utc>, SecretError> {
|
||||||
|
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) {
|
||||||
|
return Ok(dt.with_timezone(&Utc));
|
||||||
|
}
|
||||||
|
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") {
|
||||||
|
return Ok(ndt.and_utc());
|
||||||
|
}
|
||||||
|
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
|
||||||
|
return Ok(ndt.and_utc());
|
||||||
|
}
|
||||||
|
Err(SecretError::Database(format!(
|
||||||
|
"unparseable timestamp: {:?}",
|
||||||
|
s
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_row_to_secret(row: &libsql::Row) -> Result<Secret, SecretError> {
|
||||||
|
let id_str: String = row
|
||||||
|
.get(0)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let user_id: String = row
|
||||||
|
.get(1)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let name: String = row
|
||||||
|
.get(2)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let encrypted_value: Vec<u8> = row
|
||||||
|
.get(3)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let key_salt: Vec<u8> = row
|
||||||
|
.get(4)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let provider: Option<String> = row.get::<String>(5).ok().filter(|s| !s.is_empty());
|
||||||
|
let expires_at = row
|
||||||
|
.get::<String>(6)
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.and_then(|s| libsql_parse_timestamp(&s).ok());
|
||||||
|
let last_used_at = row
|
||||||
|
.get::<String>(7)
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.and_then(|s| libsql_parse_timestamp(&s).ok());
|
||||||
|
let usage_count: i64 = row.get::<i64>(8).unwrap_or(0);
|
||||||
|
let created_at_str: String = row
|
||||||
|
.get(9)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
let updated_at_str: String = row
|
||||||
|
.get(10)
|
||||||
|
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(Secret {
|
||||||
|
id: id_str
|
||||||
|
.parse()
|
||||||
|
.map_err(|e: uuid::Error| SecretError::Database(e.to_string()))?,
|
||||||
|
user_id,
|
||||||
|
name,
|
||||||
|
encrypted_value,
|
||||||
|
key_salt,
|
||||||
|
provider,
|
||||||
|
expires_at,
|
||||||
|
last_used_at,
|
||||||
|
usage_count,
|
||||||
|
created_at: libsql_parse_timestamp(&created_at_str)?,
|
||||||
|
updated_at: libsql_parse_timestamp(&updated_at_str)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// In-memory implementation for testing.
|
/// In-memory implementation for testing.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub mod testing {
|
pub mod testing {
|
||||||
@@ -364,12 +695,21 @@ pub mod testing {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn get(&self, user_id: &str, name: &str) -> Result<Secret, SecretError> {
|
async fn get(&self, user_id: &str, name: &str) -> Result<Secret, SecretError> {
|
||||||
self.secrets
|
let secret = self
|
||||||
|
.secrets
|
||||||
.read()
|
.read()
|
||||||
.await
|
.await
|
||||||
.get(&(user_id.to_string(), name.to_string()))
|
.get(&(user_id.to_string(), name.to_string()))
|
||||||
.cloned()
|
.cloned()
|
||||||
.ok_or_else(|| SecretError::NotFound(name.to_string()))
|
.ok_or_else(|| SecretError::NotFound(name.to_string()))?;
|
||||||
|
|
||||||
|
if let Some(expires_at) = secret.expires_at
|
||||||
|
&& expires_at < Utc::now()
|
||||||
|
{
|
||||||
|
return Err(SecretError::Expired);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(secret)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn get_decrypted(
|
async fn get_decrypted(
|
||||||
@@ -430,10 +770,10 @@ pub mod testing {
|
|||||||
if pattern == secret_name {
|
if pattern == secret_name {
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
if let Some(prefix) = pattern.strip_suffix('*') {
|
if let Some(prefix) = pattern.strip_suffix('*')
|
||||||
if secret_name.starts_with(prefix) {
|
&& secret_name.starts_with(prefix)
|
||||||
return Ok(true);
|
{
|
||||||
}
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(false)
|
Ok(false)
|
||||||
@@ -558,6 +898,34 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_expired_secret_returns_error() {
|
||||||
|
let store = test_store();
|
||||||
|
let expires_at = chrono::Utc::now() - chrono::Duration::hours(1);
|
||||||
|
let params = CreateSecretParams::new("expired_key", "value").with_expiry(expires_at);
|
||||||
|
|
||||||
|
store.create("user1", params).await.unwrap();
|
||||||
|
|
||||||
|
let result = store.get("user1", "expired_key").await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(matches!(
|
||||||
|
result.unwrap_err(),
|
||||||
|
crate::secrets::SecretError::Expired
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_non_expired_secret_succeeds() {
|
||||||
|
let store = test_store();
|
||||||
|
let expires_at = chrono::Utc::now() + chrono::Duration::hours(1);
|
||||||
|
let params = CreateSecretParams::new("fresh_key", "value").with_expiry(expires_at);
|
||||||
|
|
||||||
|
store.create("user1", params).await.unwrap();
|
||||||
|
|
||||||
|
let result = store.get("user1", "fresh_key").await;
|
||||||
|
assert!(result.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_user_isolation() {
|
async fn test_user_isolation() {
|
||||||
let store = test_store();
|
let store = test_store();
|
||||||
|
|||||||
+30
-72
@@ -15,6 +15,10 @@ pub struct Settings {
|
|||||||
pub onboard_completed: bool,
|
pub onboard_completed: bool,
|
||||||
|
|
||||||
// === Step 1: Database ===
|
// === Step 1: Database ===
|
||||||
|
/// Database backend: "postgres" or "libsql".
|
||||||
|
#[serde(default)]
|
||||||
|
pub database_backend: Option<String>,
|
||||||
|
|
||||||
/// Database connection URL (postgres://...).
|
/// Database connection URL (postgres://...).
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub database_url: Option<String>,
|
pub database_url: Option<String>,
|
||||||
@@ -23,6 +27,14 @@ pub struct Settings {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub database_pool_size: Option<usize>,
|
pub database_pool_size: Option<usize>,
|
||||||
|
|
||||||
|
/// Path to local libSQL database file.
|
||||||
|
#[serde(default)]
|
||||||
|
pub libsql_path: Option<String>,
|
||||||
|
|
||||||
|
/// Turso cloud URL for remote replica sync.
|
||||||
|
#[serde(default)]
|
||||||
|
pub libsql_url: Option<String>,
|
||||||
|
|
||||||
// === Step 2: Security ===
|
// === Step 2: Security ===
|
||||||
/// Source for the secrets master key.
|
/// Source for the secrets master key.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
@@ -487,14 +499,6 @@ impl Default for BuilderSettings {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Settings {
|
impl Settings {
|
||||||
/// Get the default settings file path (~/.ironclaw/settings.json).
|
|
||||||
pub fn default_path() -> PathBuf {
|
|
||||||
dirs::home_dir()
|
|
||||||
.unwrap_or_else(|| PathBuf::from("."))
|
|
||||||
.join(".ironclaw")
|
|
||||||
.join("settings.json")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Reconstruct Settings from a flat key-value map (as stored in the DB).
|
/// Reconstruct Settings from a flat key-value map (as stored in the DB).
|
||||||
///
|
///
|
||||||
/// Each key is a dotted path (e.g., "agent.name"), value is a JSONB value.
|
/// Each key is a dotted path (e.g., "agent.name"), value is a JSONB value.
|
||||||
@@ -540,50 +544,27 @@ impl Settings {
|
|||||||
map
|
map
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Get the default settings file path (~/.ironclaw/settings.json).
|
||||||
|
pub fn default_path() -> std::path::PathBuf {
|
||||||
|
dirs::home_dir()
|
||||||
|
.unwrap_or_else(|| std::path::PathBuf::from("."))
|
||||||
|
.join(".ironclaw")
|
||||||
|
.join("settings.json")
|
||||||
|
}
|
||||||
|
|
||||||
/// Load settings from disk, returning default if not found.
|
/// Load settings from disk, returning default if not found.
|
||||||
pub fn load() -> Self {
|
pub fn load() -> Self {
|
||||||
Self::load_from(&Self::default_path())
|
Self::load_from(&Self::default_path())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load settings from a specific path.
|
/// Load settings from a specific path (used by bootstrap legacy migration).
|
||||||
pub fn load_from(path: &PathBuf) -> Self {
|
pub fn load_from(path: &std::path::Path) -> Self {
|
||||||
match std::fs::read_to_string(path) {
|
match std::fs::read_to_string(path) {
|
||||||
Ok(data) => serde_json::from_str(&data).unwrap_or_default(),
|
Ok(data) => serde_json::from_str(&data).unwrap_or_default(),
|
||||||
Err(_) => Self::default(),
|
Err(_) => Self::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save settings to disk.
|
|
||||||
pub fn save(&self) -> std::io::Result<()> {
|
|
||||||
self.save_to(&Self::default_path())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Save settings to a specific path.
|
|
||||||
pub fn save_to(&self, path: &PathBuf) -> std::io::Result<()> {
|
|
||||||
// Ensure parent directory exists
|
|
||||||
if let Some(parent) = path.parent() {
|
|
||||||
std::fs::create_dir_all(parent)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
let json = serde_json::to_string_pretty(self)
|
|
||||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?;
|
|
||||||
|
|
||||||
std::fs::write(path, json)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get the selected model, falling back to the provided default.
|
|
||||||
pub fn model_or(&self, default: &str) -> String {
|
|
||||||
self.selected_model
|
|
||||||
.clone()
|
|
||||||
.unwrap_or_else(|| default.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Set the selected model and save.
|
|
||||||
pub fn set_model(&mut self, model: &str) -> std::io::Result<()> {
|
|
||||||
self.selected_model = Some(model.to_string());
|
|
||||||
self.save()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Get a setting value by dotted path (e.g., "agent.max_parallel_jobs").
|
/// Get a setting value by dotted path (e.g., "agent.max_parallel_jobs").
|
||||||
pub fn get(&self, path: &str) -> Option<String> {
|
pub fn get(&self, path: &str) -> Option<String> {
|
||||||
let json = serde_json::to_value(self).ok()?;
|
let json = serde_json::to_value(self).ok()?;
|
||||||
@@ -768,42 +749,22 @@ fn collect_settings(
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use tempfile::tempdir;
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_settings_save_load() {
|
fn test_db_map_round_trip() {
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let path = dir.path().join("settings.json");
|
|
||||||
|
|
||||||
let settings = Settings {
|
let settings = Settings {
|
||||||
selected_model: Some("claude-3-5-sonnet-20241022".to_string()),
|
selected_model: Some("claude-3-5-sonnet-20241022".to_string()),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
settings.save_to(&path).unwrap();
|
let map = settings.to_db_map();
|
||||||
|
let restored = Settings::from_db_map(&map);
|
||||||
let loaded = Settings::load_from(&path);
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
loaded.selected_model,
|
restored.selected_model,
|
||||||
Some("claude-3-5-sonnet-20241022".to_string())
|
Some("claude-3-5-sonnet-20241022".to_string())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn test_model_or_default() {
|
|
||||||
let settings = Settings::default();
|
|
||||||
assert_eq!(
|
|
||||||
settings.model_or("default-model"),
|
|
||||||
"default-model".to_string()
|
|
||||||
);
|
|
||||||
|
|
||||||
let settings = Settings {
|
|
||||||
selected_model: Some("my-model".to_string()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
assert_eq!(settings.model_or("default-model"), "my-model".to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_get_setting() {
|
fn test_get_setting() {
|
||||||
let settings = Settings::default();
|
let settings = Settings::default();
|
||||||
@@ -874,16 +835,13 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_telegram_owner_id_round_trip() {
|
fn test_telegram_owner_id_db_round_trip() {
|
||||||
let dir = tempdir().unwrap();
|
|
||||||
let path = dir.path().join("settings.json");
|
|
||||||
|
|
||||||
let mut settings = Settings::default();
|
let mut settings = Settings::default();
|
||||||
settings.channels.telegram_owner_id = Some(123456789);
|
settings.channels.telegram_owner_id = Some(123456789);
|
||||||
settings.save_to(&path).unwrap();
|
|
||||||
|
|
||||||
let loaded = Settings::load_from(&path);
|
let map = settings.to_db_map();
|
||||||
assert_eq!(loaded.channels.telegram_owner_id, Some(123456789));
|
let restored = Settings::from_db_map(&map);
|
||||||
|
assert_eq!(restored.channels.telegram_owner_id, Some(123456789));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
+58
-51
@@ -12,23 +12,34 @@ use reqwest::Client;
|
|||||||
use secrecy::{ExposeSecret, SecretString};
|
use secrecy::{ExposeSecret, SecretString};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
|
|
||||||
use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
#[cfg(feature = "postgres")]
|
||||||
use crate::settings::Settings;
|
use crate::secrets::SecretsCrypto;
|
||||||
|
use crate::secrets::{CreateSecretParams, SecretsStore};
|
||||||
|
use crate::settings::{Settings, TunnelSettings};
|
||||||
use crate::setup::prompts::{
|
use crate::setup::prompts::{
|
||||||
confirm, input, optional_input, print_error, print_info, print_success, secret_input,
|
confirm, input, optional_input, print_error, print_info, print_success, secret_input,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Context for saving secrets during setup.
|
/// Context for saving secrets during setup.
|
||||||
pub struct SecretsContext {
|
pub struct SecretsContext {
|
||||||
store: PostgresSecretsStore,
|
store: Arc<dyn SecretsStore>,
|
||||||
user_id: String,
|
user_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SecretsContext {
|
impl SecretsContext {
|
||||||
/// Create a new secrets context.
|
/// Create a new secrets context from a trait-object store.
|
||||||
|
pub fn from_store(store: Arc<dyn SecretsStore>, user_id: &str) -> Self {
|
||||||
|
Self {
|
||||||
|
store,
|
||||||
|
user_id: user_id.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a new secrets context from a PostgreSQL pool and crypto.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub fn new(pool: deadpool_postgres::Pool, crypto: Arc<SecretsCrypto>, user_id: &str) -> Self {
|
pub fn new(pool: deadpool_postgres::Pool, crypto: Arc<SecretsCrypto>, user_id: &str) -> Self {
|
||||||
Self {
|
Self {
|
||||||
store: PostgresSecretsStore::new(pool, crypto),
|
store: Arc::new(crate::secrets::PostgresSecretsStore::new(pool, crypto)),
|
||||||
user_id: user_id.to_string(),
|
user_id: user_id.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -120,7 +131,10 @@ struct TelegramUpdateUser {
|
|||||||
/// 2. Entering the bot token
|
/// 2. Entering the bot token
|
||||||
/// 3. Validating the token
|
/// 3. Validating the token
|
||||||
/// 4. Saving the token to the database
|
/// 4. Saving the token to the database
|
||||||
pub async fn setup_telegram(secrets: &SecretsContext) -> Result<TelegramSetupResult, String> {
|
pub async fn setup_telegram(
|
||||||
|
secrets: &SecretsContext,
|
||||||
|
settings: &Settings,
|
||||||
|
) -> Result<TelegramSetupResult, String> {
|
||||||
println!("Telegram Setup:");
|
println!("Telegram Setup:");
|
||||||
println!();
|
println!();
|
||||||
print_info("To create a Telegram bot:");
|
print_info("To create a Telegram bot:");
|
||||||
@@ -134,8 +148,8 @@ pub async fn setup_telegram(secrets: &SecretsContext) -> Result<TelegramSetupRes
|
|||||||
print_info("Existing Telegram token found in database.");
|
print_info("Existing Telegram token found in database.");
|
||||||
if !confirm("Replace existing token?", false).map_err(|e| e.to_string())? {
|
if !confirm("Replace existing token?", false).map_err(|e| e.to_string())? {
|
||||||
// Still offer to configure webhook secret and owner binding
|
// Still offer to configure webhook secret and owner binding
|
||||||
let webhook_secret = setup_telegram_webhook_secret(secrets).await?;
|
let webhook_secret = setup_telegram_webhook_secret(secrets, &settings.tunnel).await?;
|
||||||
let owner_id = bind_telegram_owner_flow(secrets).await?;
|
let owner_id = bind_telegram_owner_flow(secrets, settings).await?;
|
||||||
return Ok(TelegramSetupResult {
|
return Ok(TelegramSetupResult {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
bot_username: None,
|
bot_username: None,
|
||||||
@@ -165,7 +179,7 @@ pub async fn setup_telegram(secrets: &SecretsContext) -> Result<TelegramSetupRes
|
|||||||
let owner_id = bind_telegram_owner(&token).await?;
|
let owner_id = bind_telegram_owner(&token).await?;
|
||||||
|
|
||||||
// Offer webhook secret configuration
|
// Offer webhook secret configuration
|
||||||
let webhook_secret = setup_telegram_webhook_secret(secrets).await?;
|
let webhook_secret = setup_telegram_webhook_secret(secrets, &settings.tunnel).await?;
|
||||||
|
|
||||||
Ok(TelegramSetupResult {
|
Ok(TelegramSetupResult {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
@@ -178,7 +192,7 @@ pub async fn setup_telegram(secrets: &SecretsContext) -> Result<TelegramSetupRes
|
|||||||
print_error(&format!("Token validation failed: {}", e));
|
print_error(&format!("Token validation failed: {}", e));
|
||||||
|
|
||||||
if confirm("Try again?", true).map_err(|e| e.to_string())? {
|
if confirm("Try again?", true).map_err(|e| e.to_string())? {
|
||||||
Box::pin(setup_telegram(secrets)).await
|
Box::pin(setup_telegram(secrets, settings)).await
|
||||||
} else {
|
} else {
|
||||||
Ok(TelegramSetupResult {
|
Ok(TelegramSetupResult {
|
||||||
enabled: false,
|
enabled: false,
|
||||||
@@ -252,32 +266,32 @@ async fn bind_telegram_owner(token: &SecretString) -> Result<Option<i64>, String
|
|||||||
|
|
||||||
// Find the first message with a sender
|
// Find the first message with a sender
|
||||||
for update in &body.result {
|
for update in &body.result {
|
||||||
if let Some(ref msg) = update.message {
|
if let Some(ref msg) = update.message
|
||||||
if let Some(ref from) = msg.from {
|
&& let Some(ref from) = msg.from
|
||||||
let display_name = from
|
{
|
||||||
.username
|
let display_name = from
|
||||||
.as_ref()
|
.username
|
||||||
.map(|u| format!("@{}", u))
|
.as_ref()
|
||||||
.unwrap_or_else(|| from.first_name.clone());
|
.map(|u| format!("@{}", u))
|
||||||
|
.unwrap_or_else(|| from.first_name.clone());
|
||||||
|
|
||||||
print_success(&format!(
|
print_success(&format!(
|
||||||
"Received message from {} (ID: {})",
|
"Received message from {} (ID: {})",
|
||||||
display_name, from.id
|
display_name, from.id
|
||||||
));
|
));
|
||||||
|
|
||||||
// Acknowledge the update so it doesn't pile up
|
// Acknowledge the update so it doesn't pile up
|
||||||
let ack_url = format!(
|
let ack_url = format!(
|
||||||
"https://api.telegram.org/bot{}/getUpdates",
|
"https://api.telegram.org/bot{}/getUpdates",
|
||||||
token.expose_secret()
|
token.expose_secret()
|
||||||
);
|
);
|
||||||
let _ = client
|
let _ = client
|
||||||
.get(&ack_url)
|
.get(&ack_url)
|
||||||
.query(&[("offset", &(update.update_id + 1).to_string())])
|
.query(&[("offset", &(update.update_id + 1).to_string())])
|
||||||
.send()
|
.send()
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
return Ok(Some(from.id));
|
return Ok(Some(from.id));
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -290,9 +304,10 @@ async fn bind_telegram_owner(token: &SecretString) -> Result<Option<i64>, String
|
|||||||
/// Bind flow when the token already exists (reads from secrets store).
|
/// Bind flow when the token already exists (reads from secrets store).
|
||||||
///
|
///
|
||||||
/// Retrieves the saved bot token and delegates to `bind_telegram_owner`.
|
/// Retrieves the saved bot token and delegates to `bind_telegram_owner`.
|
||||||
async fn bind_telegram_owner_flow(secrets: &SecretsContext) -> Result<Option<i64>, String> {
|
async fn bind_telegram_owner_flow(
|
||||||
// Check current settings first
|
secrets: &SecretsContext,
|
||||||
let settings = Settings::load();
|
settings: &Settings,
|
||||||
|
) -> Result<Option<i64>, String> {
|
||||||
if settings.channels.telegram_owner_id.is_some() {
|
if settings.channels.telegram_owner_id.is_some() {
|
||||||
print_info("Bot is already bound to a Telegram account.");
|
print_info("Bot is already bound to a Telegram account.");
|
||||||
if !confirm("Re-bind to a different account?", false).map_err(|e| e.to_string())? {
|
if !confirm("Re-bind to a different account?", false).map_err(|e| e.to_string())? {
|
||||||
@@ -310,9 +325,7 @@ async fn bind_telegram_owner_flow(secrets: &SecretsContext) -> Result<Option<i64
|
|||||||
///
|
///
|
||||||
/// This is shared across all channels that need webhook endpoints.
|
/// This is shared across all channels that need webhook endpoints.
|
||||||
/// Returns the tunnel URL if configured.
|
/// Returns the tunnel URL if configured.
|
||||||
pub fn setup_tunnel() -> Result<Option<String>, String> {
|
pub fn setup_tunnel(settings: &Settings) -> Result<Option<String>, String> {
|
||||||
// Check if already configured
|
|
||||||
let settings = Settings::load();
|
|
||||||
if let Some(ref url) = settings.tunnel.public_url {
|
if let Some(ref url) = settings.tunnel.public_url {
|
||||||
print_info(&format!("Existing tunnel configured: {}", url));
|
print_info(&format!("Existing tunnel configured: {}", url));
|
||||||
if !confirm("Change tunnel configuration?", false).map_err(|e| e.to_string())? {
|
if !confirm("Change tunnel configuration?", false).map_err(|e| e.to_string())? {
|
||||||
@@ -351,14 +364,7 @@ pub fn setup_tunnel() -> Result<Option<String>, String> {
|
|||||||
// Remove trailing slash if present
|
// Remove trailing slash if present
|
||||||
let tunnel_url = tunnel_url.trim_end_matches('/').to_string();
|
let tunnel_url = tunnel_url.trim_end_matches('/').to_string();
|
||||||
|
|
||||||
// Save to settings
|
print_success(&format!("Tunnel URL configured: {}", tunnel_url));
|
||||||
let mut settings = Settings::load();
|
|
||||||
settings.tunnel.public_url = Some(tunnel_url.clone());
|
|
||||||
settings
|
|
||||||
.save()
|
|
||||||
.map_err(|e| format!("Failed to save settings: {}", e))?;
|
|
||||||
|
|
||||||
print_success(&format!("Tunnel URL saved: {}", tunnel_url));
|
|
||||||
print_info("");
|
print_info("");
|
||||||
print_info("Make sure your tunnel is running before starting the agent.");
|
print_info("Make sure your tunnel is running before starting the agent.");
|
||||||
print_info("You can also set TUNNEL_URL environment variable to override.");
|
print_info("You can also set TUNNEL_URL environment variable to override.");
|
||||||
@@ -369,10 +375,11 @@ pub fn setup_tunnel() -> Result<Option<String>, String> {
|
|||||||
/// Set up Telegram webhook secret for signature validation.
|
/// Set up Telegram webhook secret for signature validation.
|
||||||
///
|
///
|
||||||
/// Returns the webhook secret if configured.
|
/// Returns the webhook secret if configured.
|
||||||
async fn setup_telegram_webhook_secret(secrets: &SecretsContext) -> Result<Option<String>, String> {
|
async fn setup_telegram_webhook_secret(
|
||||||
// Check if tunnel is configured
|
secrets: &SecretsContext,
|
||||||
let settings = Settings::load();
|
tunnel: &TunnelSettings,
|
||||||
if settings.tunnel.public_url.is_none() {
|
) -> Result<Option<String>, String> {
|
||||||
|
if tunnel.public_url.is_none() {
|
||||||
print_info("");
|
print_info("");
|
||||||
print_info("No tunnel configured. Telegram will use polling mode (30s+ delay).");
|
print_info("No tunnel configured. Telegram will use polling mode (30s+ delay).");
|
||||||
print_info("Run setup again to configure a tunnel for instant delivery.");
|
print_info("Run setup again to configure a tunnel for instant delivery.");
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
|
|
||||||
mod channels;
|
mod channels;
|
||||||
mod prompts;
|
mod prompts;
|
||||||
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
mod wizard;
|
mod wizard;
|
||||||
|
|
||||||
pub use channels::{
|
pub use channels::{
|
||||||
@@ -29,4 +30,5 @@ pub use prompts::{
|
|||||||
confirm, input, optional_input, print_error, print_header, print_info, print_step,
|
confirm, input, optional_input, print_error, print_header, print_info, print_step,
|
||||||
print_success, secret_input, select_many, select_one,
|
print_success, secret_input, select_many, select_one,
|
||||||
};
|
};
|
||||||
|
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||||
pub use wizard::{SetupConfig, SetupWizard};
|
pub use wizard::{SetupConfig, SetupWizard};
|
||||||
|
|||||||
@@ -54,10 +54,11 @@ pub fn select_one(prompt: &str, options: &[&str]) -> io::Result<usize> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Parse number
|
// Parse number
|
||||||
if let Ok(num) = input.parse::<usize>() {
|
if let Ok(num) = input.parse::<usize>()
|
||||||
if num >= 1 && num <= options.len() {
|
&& num >= 1
|
||||||
return Ok(num - 1);
|
&& num <= options.len()
|
||||||
}
|
{
|
||||||
|
return Ok(num - 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
writeln!(
|
writeln!(
|
||||||
|
|||||||
+387
-64
@@ -12,15 +12,17 @@
|
|||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use deadpool_postgres::{Config as PoolConfig, Runtime};
|
use deadpool_postgres::{Config as PoolConfig, Runtime};
|
||||||
use secrecy::SecretString;
|
use secrecy::SecretString;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use tokio_postgres::NoTls;
|
use tokio_postgres::NoTls;
|
||||||
|
|
||||||
use crate::channels::wasm::{
|
use crate::channels::wasm::{
|
||||||
ChannelCapabilitiesFile, available_channel_names, install_bundled_channel,
|
ChannelCapabilitiesFile, available_channel_names, install_bundled_channel,
|
||||||
};
|
};
|
||||||
use crate::llm::{SessionConfig, SessionManager};
|
use crate::llm::{SessionConfig, SessionManager};
|
||||||
use crate::secrets::SecretsCrypto;
|
use crate::secrets::{SecretsCrypto, SecretsStore};
|
||||||
use crate::settings::{KeySource, Settings};
|
use crate::settings::{KeySource, Settings};
|
||||||
use crate::setup::channels::{
|
use crate::setup::channels::{
|
||||||
SecretsContext, setup_http, setup_telegram, setup_tunnel, setup_wasm_channel,
|
SecretsContext, setup_http, setup_telegram, setup_tunnel, setup_wasm_channel,
|
||||||
@@ -66,8 +68,12 @@ pub struct SetupWizard {
|
|||||||
config: SetupConfig,
|
config: SetupConfig,
|
||||||
settings: Settings,
|
settings: Settings,
|
||||||
session_manager: Option<Arc<SessionManager>>,
|
session_manager: Option<Arc<SessionManager>>,
|
||||||
/// Database pool (created during setup).
|
/// Database pool (created during setup, postgres only).
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
db_pool: Option<deadpool_postgres::Pool>,
|
db_pool: Option<deadpool_postgres::Pool>,
|
||||||
|
/// libSQL backend (created during setup, libsql only).
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
db_backend: Option<crate::db::libsql_backend::LibSqlBackend>,
|
||||||
/// Secrets crypto (created during setup).
|
/// Secrets crypto (created during setup).
|
||||||
secrets_crypto: Option<Arc<SecretsCrypto>>,
|
secrets_crypto: Option<Arc<SecretsCrypto>>,
|
||||||
}
|
}
|
||||||
@@ -77,9 +83,12 @@ impl SetupWizard {
|
|||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
config: SetupConfig::default(),
|
config: SetupConfig::default(),
|
||||||
settings: Settings::load(),
|
settings: Settings::default(),
|
||||||
session_manager: None,
|
session_manager: None,
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
db_pool: None,
|
db_pool: None,
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
db_backend: None,
|
||||||
secrets_crypto: None,
|
secrets_crypto: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -88,9 +97,12 @@ impl SetupWizard {
|
|||||||
pub fn with_config(config: SetupConfig) -> Self {
|
pub fn with_config(config: SetupConfig) -> Self {
|
||||||
Self {
|
Self {
|
||||||
config,
|
config,
|
||||||
settings: Settings::load(),
|
settings: Settings::default(),
|
||||||
session_manager: None,
|
session_manager: None,
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
db_pool: None,
|
db_pool: None,
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
db_backend: None,
|
||||||
secrets_crypto: None,
|
secrets_crypto: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -146,26 +158,55 @@ impl SetupWizard {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Save settings and print summary
|
// Save settings and print summary
|
||||||
self.save_and_summarize()?;
|
self.save_and_summarize().await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Step 1: Database connection.
|
/// Step 1: Database connection.
|
||||||
async fn step_database(&mut self) -> Result<(), SetupError> {
|
async fn step_database(&mut self) -> Result<(), SetupError> {
|
||||||
// Check if we have an existing URL in env or settings
|
// Determine which backend to use based on compile-time features.
|
||||||
|
// When both features are enabled, prefer the currently configured backend
|
||||||
|
// or default to postgres.
|
||||||
|
#[cfg(all(feature = "postgres", feature = "libsql"))]
|
||||||
|
{
|
||||||
|
let backend = std::env::var("DATABASE_BACKEND")
|
||||||
|
.ok()
|
||||||
|
.or_else(|| self.settings.database_backend.clone())
|
||||||
|
.unwrap_or_else(|| "postgres".to_string());
|
||||||
|
|
||||||
|
if backend == "libsql" || backend == "turso" || backend == "sqlite" {
|
||||||
|
return self.step_database_libsql().await;
|
||||||
|
}
|
||||||
|
return self.step_database_postgres().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "postgres", not(feature = "libsql")))]
|
||||||
|
{
|
||||||
|
return self.step_database_postgres().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||||
|
{
|
||||||
|
return self.step_database_libsql().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Step 1 (postgres): Database connection via PostgreSQL URL.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
async fn step_database_postgres(&mut self) -> Result<(), SetupError> {
|
||||||
|
self.settings.database_backend = Some("postgres".to_string());
|
||||||
|
|
||||||
let existing_url = std::env::var("DATABASE_URL")
|
let existing_url = std::env::var("DATABASE_URL")
|
||||||
.ok()
|
.ok()
|
||||||
.or_else(|| self.settings.database_url.clone());
|
.or_else(|| self.settings.database_url.clone());
|
||||||
|
|
||||||
if let Some(ref url) = existing_url {
|
if let Some(ref url) = existing_url {
|
||||||
// Mask the password for display
|
|
||||||
let display_url = mask_password_in_url(url);
|
let display_url = mask_password_in_url(url);
|
||||||
print_info(&format!("Existing database URL: {}", display_url));
|
print_info(&format!("Existing database URL: {}", display_url));
|
||||||
|
|
||||||
if confirm("Use this database?", true).map_err(SetupError::Io)? {
|
if confirm("Use this database?", true).map_err(SetupError::Io)? {
|
||||||
// Test the connection
|
if let Err(e) = self.test_database_connection_postgres(url).await {
|
||||||
if let Err(e) = self.test_database_connection(url).await {
|
|
||||||
print_error(&format!("Connection failed: {}", e));
|
print_error(&format!("Connection failed: {}", e));
|
||||||
print_info("Let's configure a new database URL.");
|
print_info("Let's configure a new database URL.");
|
||||||
} else {
|
} else {
|
||||||
@@ -176,7 +217,6 @@ impl SetupWizard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prompt for new URL
|
|
||||||
println!();
|
println!();
|
||||||
print_info("Enter your PostgreSQL connection URL.");
|
print_info("Enter your PostgreSQL connection URL.");
|
||||||
print_info("Format: postgres://user:password@host:port/database");
|
print_info("Format: postgres://user:password@host:port/database");
|
||||||
@@ -190,15 +230,13 @@ impl SetupWizard {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test the connection
|
|
||||||
print_info("Testing connection...");
|
print_info("Testing connection...");
|
||||||
match self.test_database_connection(&url).await {
|
match self.test_database_connection_postgres(&url).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
print_success("Database connection successful");
|
print_success("Database connection successful");
|
||||||
|
|
||||||
// Ask if we should run migrations
|
|
||||||
if confirm("Run database migrations?", true).map_err(SetupError::Io)? {
|
if confirm("Run database migrations?", true).map_err(SetupError::Io)? {
|
||||||
self.run_migrations().await?;
|
self.run_migrations_postgres().await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
self.settings.database_url = Some(url);
|
self.settings.database_url = Some(url);
|
||||||
@@ -216,8 +254,115 @@ impl SetupWizard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Test database connection and store the pool.
|
/// Step 1 (libsql): Database connection via local file or Turso remote replica.
|
||||||
async fn test_database_connection(&mut self, url: &str) -> Result<(), SetupError> {
|
#[cfg(feature = "libsql")]
|
||||||
|
async fn step_database_libsql(&mut self) -> Result<(), SetupError> {
|
||||||
|
self.settings.database_backend = Some("libsql".to_string());
|
||||||
|
|
||||||
|
let default_path = crate::config::default_libsql_path();
|
||||||
|
let default_path_str = default_path.to_string_lossy().to_string();
|
||||||
|
|
||||||
|
// Check for existing configuration
|
||||||
|
let existing_path = std::env::var("LIBSQL_PATH")
|
||||||
|
.ok()
|
||||||
|
.or_else(|| self.settings.libsql_path.clone());
|
||||||
|
|
||||||
|
if let Some(ref path) = existing_path {
|
||||||
|
print_info(&format!("Existing database path: {}", path));
|
||||||
|
if confirm("Use this database?", true).map_err(SetupError::Io)? {
|
||||||
|
let turso_url = std::env::var("LIBSQL_URL")
|
||||||
|
.ok()
|
||||||
|
.or_else(|| self.settings.libsql_url.clone());
|
||||||
|
let turso_token = std::env::var("LIBSQL_AUTH_TOKEN").ok();
|
||||||
|
|
||||||
|
match self
|
||||||
|
.test_database_connection_libsql(
|
||||||
|
path,
|
||||||
|
turso_url.as_deref(),
|
||||||
|
turso_token.as_deref(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(()) => {
|
||||||
|
print_success("Database connection successful");
|
||||||
|
self.settings.libsql_path = Some(path.clone());
|
||||||
|
if let Some(url) = turso_url {
|
||||||
|
self.settings.libsql_url = Some(url);
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
print_error(&format!("Connection failed: {}", e));
|
||||||
|
print_info("Let's configure a new database path.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!();
|
||||||
|
print_info("IronClaw uses an embedded SQLite database (libSQL).");
|
||||||
|
print_info("No external database server required.");
|
||||||
|
println!();
|
||||||
|
|
||||||
|
let path_input = optional_input(
|
||||||
|
"Database file path",
|
||||||
|
Some(&format!("default: {}", default_path_str)),
|
||||||
|
)
|
||||||
|
.map_err(SetupError::Io)?;
|
||||||
|
|
||||||
|
let db_path = path_input.unwrap_or(default_path_str.clone());
|
||||||
|
|
||||||
|
// Ask about Turso cloud sync
|
||||||
|
println!();
|
||||||
|
let use_turso =
|
||||||
|
confirm("Enable Turso cloud sync (remote replica)?", false).map_err(SetupError::Io)?;
|
||||||
|
|
||||||
|
let (turso_url, turso_token) = if use_turso {
|
||||||
|
print_info("Enter your Turso database URL and auth token.");
|
||||||
|
print_info("Format: libsql://your-db.turso.io");
|
||||||
|
println!();
|
||||||
|
|
||||||
|
let url = input("Turso URL").map_err(SetupError::Io)?;
|
||||||
|
if url.is_empty() {
|
||||||
|
print_error("Turso URL is required for cloud sync.");
|
||||||
|
(None, None)
|
||||||
|
} else {
|
||||||
|
let token = input("Auth token").map_err(SetupError::Io)?;
|
||||||
|
if token.is_empty() {
|
||||||
|
print_error("Auth token is required for cloud sync.");
|
||||||
|
(None, None)
|
||||||
|
} else {
|
||||||
|
(Some(url), Some(token))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(None, None)
|
||||||
|
};
|
||||||
|
|
||||||
|
print_info("Testing connection...");
|
||||||
|
match self
|
||||||
|
.test_database_connection_libsql(&db_path, turso_url.as_deref(), turso_token.as_deref())
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(()) => {
|
||||||
|
print_success("Database connection successful");
|
||||||
|
|
||||||
|
// Always run migrations for libsql (they're idempotent)
|
||||||
|
self.run_migrations_libsql().await?;
|
||||||
|
|
||||||
|
self.settings.libsql_path = Some(db_path);
|
||||||
|
if let Some(url) = turso_url {
|
||||||
|
self.settings.libsql_url = Some(url);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
Err(e) => Err(SetupError::Database(format!("Connection failed: {}", e))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Test PostgreSQL connection and store the pool.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
async fn test_database_connection_postgres(&mut self, url: &str) -> Result<(), SetupError> {
|
||||||
let mut cfg = PoolConfig::new();
|
let mut cfg = PoolConfig::new();
|
||||||
cfg.url = Some(url.to_string());
|
cfg.url = Some(url.to_string());
|
||||||
cfg.pool = Some(deadpool_postgres::PoolConfig {
|
cfg.pool = Some(deadpool_postgres::PoolConfig {
|
||||||
@@ -229,7 +374,6 @@ impl SetupWizard {
|
|||||||
.create_pool(Some(Runtime::Tokio1), NoTls)
|
.create_pool(Some(Runtime::Tokio1), NoTls)
|
||||||
.map_err(|e| SetupError::Database(format!("Failed to create pool: {}", e)))?;
|
.map_err(|e| SetupError::Database(format!("Failed to create pool: {}", e)))?;
|
||||||
|
|
||||||
// Test the connection
|
|
||||||
let _ = pool
|
let _ = pool
|
||||||
.get()
|
.get()
|
||||||
.await
|
.await
|
||||||
@@ -239,8 +383,36 @@ impl SetupWizard {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run database migrations.
|
/// Test libSQL connection and store the backend.
|
||||||
async fn run_migrations(&self) -> Result<(), SetupError> {
|
#[cfg(feature = "libsql")]
|
||||||
|
async fn test_database_connection_libsql(
|
||||||
|
&mut self,
|
||||||
|
path: &str,
|
||||||
|
turso_url: Option<&str>,
|
||||||
|
turso_token: Option<&str>,
|
||||||
|
) -> Result<(), SetupError> {
|
||||||
|
use crate::db::libsql_backend::LibSqlBackend;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
let db_path = Path::new(path);
|
||||||
|
|
||||||
|
let backend = if let (Some(url), Some(token)) = (turso_url, turso_token) {
|
||||||
|
LibSqlBackend::new_remote_replica(db_path, url, token)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SetupError::Database(format!("Failed to connect: {}", e)))?
|
||||||
|
} else {
|
||||||
|
LibSqlBackend::new_local(db_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| SetupError::Database(format!("Failed to open database: {}", e)))?
|
||||||
|
};
|
||||||
|
|
||||||
|
self.db_backend = Some(backend);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run PostgreSQL migrations.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
async fn run_migrations_postgres(&self) -> Result<(), SetupError> {
|
||||||
if let Some(ref pool) = self.db_pool {
|
if let Some(ref pool) = self.db_pool {
|
||||||
use refinery::embed_migrations;
|
use refinery::embed_migrations;
|
||||||
embed_migrations!("migrations");
|
embed_migrations!("migrations");
|
||||||
@@ -262,6 +434,24 @@ impl SetupWizard {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run libSQL migrations.
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
async fn run_migrations_libsql(&self) -> Result<(), SetupError> {
|
||||||
|
if let Some(ref backend) = self.db_backend {
|
||||||
|
use crate::db::Database;
|
||||||
|
|
||||||
|
print_info("Running migrations...");
|
||||||
|
|
||||||
|
backend
|
||||||
|
.run_migrations()
|
||||||
|
.await
|
||||||
|
.map_err(|e| SetupError::Database(format!("Migration failed: {}", e)))?;
|
||||||
|
|
||||||
|
print_success("Migrations applied");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Step 2: Security (secrets master key).
|
/// Step 2: Security (secrets master key).
|
||||||
async fn step_security(&mut self) -> Result<(), SetupError> {
|
async fn step_security(&mut self) -> Result<(), SetupError> {
|
||||||
// Check current configuration
|
// Check current configuration
|
||||||
@@ -344,17 +534,17 @@ impl SetupWizard {
|
|||||||
/// Step 3: NEAR AI authentication.
|
/// Step 3: NEAR AI authentication.
|
||||||
async fn step_authentication(&mut self) -> Result<(), SetupError> {
|
async fn step_authentication(&mut self) -> Result<(), SetupError> {
|
||||||
// Check if we already have a session
|
// Check if we already have a session
|
||||||
if let Some(ref session) = self.session_manager {
|
if let Some(ref session) = self.session_manager
|
||||||
if session.has_token().await {
|
&& session.has_token().await
|
||||||
print_info("Existing session found. Validating...");
|
{
|
||||||
match session.ensure_authenticated().await {
|
print_info("Existing session found. Validating...");
|
||||||
Ok(()) => {
|
match session.ensure_authenticated().await {
|
||||||
print_success("Session valid");
|
Ok(()) => {
|
||||||
return Ok(());
|
print_success("Session valid");
|
||||||
}
|
return Ok(());
|
||||||
Err(e) => {
|
}
|
||||||
print_info(&format!("Session invalid: {}. Re-authenticating...", e));
|
Err(e) => {
|
||||||
}
|
print_info(&format!("Session invalid: {}. Re-authenticating...", e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -463,6 +653,8 @@ impl SetupWizard {
|
|||||||
session_path: crate::llm::session::default_session_path(),
|
session_path: crate::llm::session::default_session_path(),
|
||||||
api_mode: crate::config::NearAiApiMode::Responses,
|
api_mode: crate::config::NearAiApiMode::Responses,
|
||||||
api_key: None,
|
api_key: None,
|
||||||
|
fallback_model: None,
|
||||||
|
max_retries: 3,
|
||||||
},
|
},
|
||||||
openai: None,
|
openai: None,
|
||||||
anthropic: None,
|
anthropic: None,
|
||||||
@@ -532,24 +724,6 @@ impl SetupWizard {
|
|||||||
|
|
||||||
/// Initialize secrets context for channel setup.
|
/// Initialize secrets context for channel setup.
|
||||||
async fn init_secrets_context(&mut self) -> Result<SecretsContext, SetupError> {
|
async fn init_secrets_context(&mut self) -> Result<SecretsContext, SetupError> {
|
||||||
// Get database pool (should be set from step 1)
|
|
||||||
let pool = if let Some(ref p) = self.db_pool {
|
|
||||||
p.clone()
|
|
||||||
} else {
|
|
||||||
// Fall back to creating one from settings/env
|
|
||||||
let url = self
|
|
||||||
.settings
|
|
||||||
.database_url
|
|
||||||
.clone()
|
|
||||||
.or_else(|| std::env::var("DATABASE_URL").ok())
|
|
||||||
.ok_or_else(|| SetupError::Config("Database URL not configured".to_string()))?;
|
|
||||||
|
|
||||||
self.test_database_connection(&url).await?;
|
|
||||||
// Ensure secrets-related tables exist for channels-only onboarding flows.
|
|
||||||
self.run_migrations().await?;
|
|
||||||
self.db_pool.clone().unwrap()
|
|
||||||
};
|
|
||||||
|
|
||||||
// Get crypto (should be set from step 2, or load from keychain/env)
|
// Get crypto (should be set from step 2, or load from keychain/env)
|
||||||
let crypto = if let Some(ref c) = self.secrets_crypto {
|
let crypto = if let Some(ref c) = self.secrets_crypto {
|
||||||
Arc::clone(c)
|
Arc::clone(c)
|
||||||
@@ -571,13 +745,80 @@ impl SetupWizard {
|
|||||||
Arc::clone(self.secrets_crypto.as_ref().unwrap())
|
Arc::clone(self.secrets_crypto.as_ref().unwrap())
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(SecretsContext::new(pool, crypto, "default"))
|
// Create backend-appropriate secrets store
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
{
|
||||||
|
// Try postgres path first when postgres feature is available
|
||||||
|
if let Some(store) = self.create_postgres_secrets_store(&crypto).await? {
|
||||||
|
return Ok(SecretsContext::from_store(store, "default"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
{
|
||||||
|
if let Some(store) = self.create_libsql_secrets_store(&crypto)? {
|
||||||
|
return Ok(SecretsContext::from_store(store, "default"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(SetupError::Config(
|
||||||
|
"No database backend available for secrets storage".to_string(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a PostgreSQL secrets store from the current pool.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
async fn create_postgres_secrets_store(
|
||||||
|
&mut self,
|
||||||
|
crypto: &Arc<SecretsCrypto>,
|
||||||
|
) -> Result<Option<Arc<dyn SecretsStore>>, SetupError> {
|
||||||
|
let pool = if let Some(ref p) = self.db_pool {
|
||||||
|
p.clone()
|
||||||
|
} else {
|
||||||
|
// Fall back to creating one from settings/env
|
||||||
|
let url = self
|
||||||
|
.settings
|
||||||
|
.database_url
|
||||||
|
.clone()
|
||||||
|
.or_else(|| std::env::var("DATABASE_URL").ok());
|
||||||
|
|
||||||
|
if let Some(url) = url {
|
||||||
|
self.test_database_connection_postgres(&url).await?;
|
||||||
|
self.run_migrations_postgres().await?;
|
||||||
|
self.db_pool.clone().unwrap()
|
||||||
|
} else {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let store: Arc<dyn SecretsStore> = Arc::new(crate::secrets::PostgresSecretsStore::new(
|
||||||
|
pool,
|
||||||
|
Arc::clone(crypto),
|
||||||
|
));
|
||||||
|
Ok(Some(store))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Create a libSQL secrets store from the current backend.
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn create_libsql_secrets_store(
|
||||||
|
&self,
|
||||||
|
crypto: &Arc<SecretsCrypto>,
|
||||||
|
) -> Result<Option<Arc<dyn SecretsStore>>, SetupError> {
|
||||||
|
if let Some(ref backend) = self.db_backend {
|
||||||
|
let store: Arc<dyn SecretsStore> = Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||||
|
backend.shared_db(),
|
||||||
|
Arc::clone(crypto),
|
||||||
|
));
|
||||||
|
Ok(Some(store))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Step 6: Channel configuration.
|
/// Step 6: Channel configuration.
|
||||||
async fn step_channels(&mut self) -> Result<(), SetupError> {
|
async fn step_channels(&mut self) -> Result<(), SetupError> {
|
||||||
// First, configure tunnel (shared across all channels that need webhooks)
|
// First, configure tunnel (shared across all channels that need webhooks)
|
||||||
match setup_tunnel() {
|
match setup_tunnel(&self.settings) {
|
||||||
Ok(Some(url)) => {
|
Ok(Some(url)) => {
|
||||||
self.settings.tunnel.public_url = Some(url);
|
self.settings.tunnel.public_url = Some(url);
|
||||||
}
|
}
|
||||||
@@ -642,11 +883,10 @@ impl SetupWizard {
|
|||||||
&installed_names,
|
&installed_names,
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
|
&& !installed.is_empty()
|
||||||
{
|
{
|
||||||
if !installed.is_empty() {
|
print_success(&format!("Installed channels: {}", installed.join(", ")));
|
||||||
print_success(&format!("Installed channels: {}", installed.join(", ")));
|
discovered_channels = discover_wasm_channels(&channels_dir).await;
|
||||||
discovered_channels = discover_wasm_channels(&channels_dir).await;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Determine if we need secrets context
|
// Determine if we need secrets context
|
||||||
@@ -694,8 +934,9 @@ impl SetupWizard {
|
|||||||
.await
|
.await
|
||||||
.map_err(SetupError::Channel)?
|
.map_err(SetupError::Channel)?
|
||||||
} else if channel_name == "telegram" {
|
} else if channel_name == "telegram" {
|
||||||
let telegram_result =
|
let telegram_result = setup_telegram(ctx, &self.settings)
|
||||||
setup_telegram(ctx).await.map_err(SetupError::Channel)?;
|
.await
|
||||||
|
.map_err(SetupError::Channel)?;
|
||||||
if let Some(owner_id) = telegram_result.owner_id {
|
if let Some(owner_id) = telegram_result.owner_id {
|
||||||
self.settings.channels.telegram_owner_id = Some(owner_id);
|
self.settings.channels.telegram_owner_id = Some(owner_id);
|
||||||
}
|
}
|
||||||
@@ -778,24 +1019,104 @@ impl SetupWizard {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Save settings and print summary.
|
/// Save settings to the database and `~/.ironclaw/.env`, then print summary.
|
||||||
fn save_and_summarize(&mut self) -> Result<(), SetupError> {
|
async fn save_and_summarize(&mut self) -> Result<(), SetupError> {
|
||||||
self.settings.onboard_completed = true;
|
self.settings.onboard_completed = true;
|
||||||
|
|
||||||
self.settings
|
// Write all settings to the database (whichever backend is active).
|
||||||
.save()
|
{
|
||||||
.map_err(|e| std::io::Error::other(format!("Failed to save settings: {}", e)))?;
|
let db_map = self.settings.to_db_map();
|
||||||
|
let saved = false;
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
let saved = if !saved {
|
||||||
|
if let Some(ref pool) = self.db_pool {
|
||||||
|
let store = crate::history::Store::from_pool(pool.clone());
|
||||||
|
store
|
||||||
|
.set_all_settings("default", &db_map)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
SetupError::Database(format!(
|
||||||
|
"Failed to save settings to database: {}",
|
||||||
|
e
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
saved
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
let saved = if !saved {
|
||||||
|
if let Some(ref backend) = self.db_backend {
|
||||||
|
use crate::db::Database as _;
|
||||||
|
backend
|
||||||
|
.set_all_settings("default", &db_map)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
SetupError::Database(format!(
|
||||||
|
"Failed to save settings to database: {}",
|
||||||
|
e
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
saved
|
||||||
|
};
|
||||||
|
|
||||||
|
if !saved {
|
||||||
|
return Err(SetupError::Database(
|
||||||
|
"No database connection, cannot save settings".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save DATABASE_URL to ~/.ironclaw/.env (the only field that needs
|
||||||
|
// disk persistence before the DB is available).
|
||||||
|
if let Some(ref url) = self.settings.database_url {
|
||||||
|
crate::bootstrap::save_database_url(url).map_err(|e| {
|
||||||
|
SetupError::Io(std::io::Error::other(format!(
|
||||||
|
"Failed to save DATABASE_URL to .env: {}",
|
||||||
|
e
|
||||||
|
)))
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
println!();
|
println!();
|
||||||
print_success("Configuration saved to ~/.ironclaw/");
|
print_success("Configuration saved to database");
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
// Print summary
|
// Print summary
|
||||||
println!("Configuration Summary:");
|
println!("Configuration Summary:");
|
||||||
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
||||||
|
|
||||||
if self.settings.database_url.is_some() {
|
let backend = self
|
||||||
println!(" Database: configured");
|
.settings
|
||||||
|
.database_backend
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or("postgres");
|
||||||
|
match backend {
|
||||||
|
"libsql" => {
|
||||||
|
if let Some(ref path) = self.settings.libsql_path {
|
||||||
|
println!(" Database: libSQL ({})", path);
|
||||||
|
} else {
|
||||||
|
println!(" Database: libSQL (default path)");
|
||||||
|
}
|
||||||
|
if self.settings.libsql_url.is_some() {
|
||||||
|
println!(" Turso sync: enabled");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
if self.settings.database_url.is_some() {
|
||||||
|
println!(" Database: PostgreSQL (configured)");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
match self.settings.secrets_master_key_source {
|
match self.settings.secrets_master_key_source {
|
||||||
@@ -875,6 +1196,7 @@ impl Default for SetupWizard {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Mask password in a database URL for display.
|
/// Mask password in a database URL for display.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn mask_password_in_url(url: &str) -> String {
|
fn mask_password_in_url(url: &str) -> String {
|
||||||
// URL format: scheme://user:password@host/database
|
// URL format: scheme://user:password@host/database
|
||||||
// Find "://" to locate start of credentials
|
// Find "://" to locate start of credentials
|
||||||
@@ -1065,6 +1387,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn test_mask_password_in_url() {
|
fn test_mask_password_in_url() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
mask_password_in_url("postgres://user:secret@localhost/db"),
|
mask_password_in_url("postgres://user:secret@localhost/db"),
|
||||||
|
|||||||
@@ -326,20 +326,20 @@ impl TestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Verify expected output
|
// Verify expected output
|
||||||
if let Some(ref expected) = test.expected_output {
|
if let Some(ref expected) = test.expected_output
|
||||||
if &actual != expected {
|
&& &actual != expected
|
||||||
return TestResult {
|
{
|
||||||
name: test.name.clone(),
|
return TestResult {
|
||||||
passed: false,
|
name: test.name.clone(),
|
||||||
duration,
|
passed: false,
|
||||||
error: Some(format!(
|
duration,
|
||||||
"Output mismatch:\nExpected: {}\nActual: {}",
|
error: Some(format!(
|
||||||
serde_json::to_string_pretty(expected).unwrap_or_default(),
|
"Output mismatch:\nExpected: {}\nActual: {}",
|
||||||
serde_json::to_string_pretty(&actual).unwrap_or_default()
|
serde_json::to_string_pretty(expected).unwrap_or_default(),
|
||||||
)),
|
serde_json::to_string_pretty(&actual).unwrap_or_default()
|
||||||
actual_output: Some(actual),
|
)),
|
||||||
};
|
actual_output: Some(actual),
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify expected fields
|
// Verify expected fields
|
||||||
@@ -357,19 +357,19 @@ impl TestHarness {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(ref expected_value) = field.value {
|
if let Some(ref expected_value) = field.value
|
||||||
if field_value != Some(expected_value) {
|
&& field_value != Some(expected_value)
|
||||||
return TestResult {
|
{
|
||||||
name: test.name.clone(),
|
return TestResult {
|
||||||
passed: false,
|
name: test.name.clone(),
|
||||||
duration,
|
passed: false,
|
||||||
error: Some(format!(
|
duration,
|
||||||
"Field '{}' mismatch: expected {:?}, got {:?}",
|
error: Some(format!(
|
||||||
field.path, expected_value, field_value
|
"Field '{}' mismatch: expected {:?}, got {:?}",
|
||||||
)),
|
field.path, expected_value, field_value
|
||||||
actual_output: Some(actual),
|
)),
|
||||||
};
|
actual_output: Some(actual),
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,451 @@
|
|||||||
|
//! Accessibility tree parsing and element reference generation.
|
||||||
|
//!
|
||||||
|
//! Converts Chrome's CDP accessibility tree into a compact, LLM-friendly
|
||||||
|
//! representation with stable element references (`@e1`, `@e2`, ...).
|
||||||
|
//!
|
||||||
|
//! The key insight: sending the full accessibility tree every turn is wasteful.
|
||||||
|
//! Instead, we assign short IDs to interactive elements and let the LLM
|
||||||
|
//! reference them by ID for clicks/typing. This is ~93% cheaper in tokens
|
||||||
|
//! compared to re-sending the full tree each time.
|
||||||
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! Page: https://example.com/login
|
||||||
|
//! @e1: textbox "Email" [focused]
|
||||||
|
//! @e2: textbox "Password" [type=password]
|
||||||
|
//! @e3: button "Sign In"
|
||||||
|
//! @e4: link "Forgot password?"
|
||||||
|
//! ```
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
|
use chromiumoxide::cdp::browser_protocol::accessibility::{AxNode, AxPropertyName};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::dom::BackendNodeId;
|
||||||
|
|
||||||
|
/// A resolved element reference that maps `@eN` back to a DOM target.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ElementRef {
|
||||||
|
/// The display label shown to the LLM (e.g., `textbox "Email"`).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub label: String,
|
||||||
|
/// CDP backend node ID for targeting this element.
|
||||||
|
pub backend_node_id: BackendNodeId,
|
||||||
|
/// CSS selector hint (best-effort, may not be unique).
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub selector_hint: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stores the current set of element references for a page snapshot.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct ElementRefMap {
|
||||||
|
refs: HashMap<String, ElementRef>,
|
||||||
|
counter: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ElementRefMap {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Look up a reference like `@e1` or just `e1`.
|
||||||
|
pub fn get(&self, ref_id: &str) -> Option<&ElementRef> {
|
||||||
|
let normalized = ref_id.strip_prefix('@').unwrap_or(ref_id);
|
||||||
|
self.refs.get(normalized)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Number of tracked elements.
|
||||||
|
#[allow(dead_code)]
|
||||||
|
pub fn len(&self) -> usize {
|
||||||
|
self.refs.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
self.refs.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reset all refs. Called before each new `read_page` and when switching tabs.
|
||||||
|
pub fn reset(&mut self) {
|
||||||
|
self.refs.clear();
|
||||||
|
self.counter = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Allocate the next reference ID and store the element.
|
||||||
|
fn insert(&mut self, elem: ElementRef) -> String {
|
||||||
|
self.counter += 1;
|
||||||
|
let id = format!("e{}", self.counter);
|
||||||
|
self.refs.insert(id.clone(), elem);
|
||||||
|
id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which elements to include when building the tree representation.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum ElementFilter {
|
||||||
|
/// Only interactive elements (buttons, links, inputs, selects, textareas).
|
||||||
|
Interactive,
|
||||||
|
/// All elements with meaningful content.
|
||||||
|
All,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ElementFilter {
|
||||||
|
pub fn from_str_opt(s: Option<&str>) -> Self {
|
||||||
|
match s {
|
||||||
|
Some("all") => Self::All,
|
||||||
|
_ => Self::Interactive,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Roles that are considered "interactive" for filtering purposes.
|
||||||
|
const INTERACTIVE_ROLES: &[&str] = &[
|
||||||
|
"button",
|
||||||
|
"link",
|
||||||
|
"textbox",
|
||||||
|
"searchbox",
|
||||||
|
"combobox",
|
||||||
|
"listbox",
|
||||||
|
"option",
|
||||||
|
"menuitem",
|
||||||
|
"menuitemcheckbox",
|
||||||
|
"menuitemradio",
|
||||||
|
"radio",
|
||||||
|
"checkbox",
|
||||||
|
"switch",
|
||||||
|
"slider",
|
||||||
|
"spinbutton",
|
||||||
|
"tab",
|
||||||
|
"treeitem",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Roles to skip entirely (structural noise).
|
||||||
|
const SKIP_ROLES: &[&str] = &[
|
||||||
|
"none",
|
||||||
|
"presentation",
|
||||||
|
"generic",
|
||||||
|
"InlineTextBox",
|
||||||
|
"LineBreak",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Build a compact page representation from the CDP accessibility tree.
|
||||||
|
///
|
||||||
|
/// Returns the text representation and populates `ref_map` with element
|
||||||
|
/// references the LLM can use for subsequent actions.
|
||||||
|
pub fn build_page_repr(
|
||||||
|
url: &str,
|
||||||
|
title: &str,
|
||||||
|
nodes: &[AxNode],
|
||||||
|
filter: ElementFilter,
|
||||||
|
ref_map: &mut ElementRefMap,
|
||||||
|
) -> String {
|
||||||
|
ref_map.reset();
|
||||||
|
|
||||||
|
let mut lines = Vec::new();
|
||||||
|
|
||||||
|
// Header
|
||||||
|
lines.push(format!("Page: {}", url));
|
||||||
|
if !title.is_empty() {
|
||||||
|
lines.push(format!("Title: {}", title));
|
||||||
|
}
|
||||||
|
lines.push(String::new());
|
||||||
|
|
||||||
|
// Walk nodes, collecting elements that pass the filter.
|
||||||
|
for node in nodes {
|
||||||
|
let role = node_role(node);
|
||||||
|
|
||||||
|
if SKIP_ROLES.contains(&role.as_str()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// For "interactive" filter, only include interactive roles.
|
||||||
|
if filter == ElementFilter::Interactive && !INTERACTIVE_ROLES.contains(&role.as_str()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Skip nodes without a name (usually decorative).
|
||||||
|
let name = node_name(node);
|
||||||
|
if name.is_empty() && filter == ElementFilter::Interactive {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let backend_id = match node.backend_dom_node_id {
|
||||||
|
Some(id) => id,
|
||||||
|
None => continue,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build display label
|
||||||
|
let mut label = NodeLabel {
|
||||||
|
role: role.clone(),
|
||||||
|
name: truncate_name(&name, 80),
|
||||||
|
properties: Vec::new(),
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add useful properties
|
||||||
|
if node_has_property(node, "focused") {
|
||||||
|
label.properties.push("focused".to_string());
|
||||||
|
}
|
||||||
|
if node_has_property(node, "checked") {
|
||||||
|
label.properties.push("checked".to_string());
|
||||||
|
}
|
||||||
|
if node_has_property(node, "disabled") {
|
||||||
|
label.properties.push("disabled".to_string());
|
||||||
|
}
|
||||||
|
if node_has_property(node, "expanded") {
|
||||||
|
label.properties.push("expanded".to_string());
|
||||||
|
}
|
||||||
|
if node_has_property(node, "required") {
|
||||||
|
label.properties.push("required".to_string());
|
||||||
|
}
|
||||||
|
if let Some(val) = node_value(node) {
|
||||||
|
if !val.is_empty() && val != name {
|
||||||
|
label
|
||||||
|
.properties
|
||||||
|
.push(format!("value=\"{}\"", truncate_name(&val, 40)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let display = label.to_string();
|
||||||
|
|
||||||
|
let elem_ref = ElementRef {
|
||||||
|
label: display.clone(),
|
||||||
|
backend_node_id: backend_id,
|
||||||
|
selector_hint: guess_selector(node),
|
||||||
|
};
|
||||||
|
|
||||||
|
let ref_id = ref_map.insert(elem_ref);
|
||||||
|
lines.push(format!("@{}: {}", ref_id, display));
|
||||||
|
}
|
||||||
|
|
||||||
|
if ref_map.is_empty() {
|
||||||
|
lines.push("(no interactive elements found)".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
lines.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the role string from an AX node.
|
||||||
|
fn node_role(node: &AxNode) -> String {
|
||||||
|
node.role
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.value.as_ref())
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("unknown")
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the name (accessible label) from an AX node.
|
||||||
|
fn node_name(node: &AxNode) -> String {
|
||||||
|
node.name
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.value.as_ref())
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the value from an AX node (for inputs, etc.).
|
||||||
|
fn node_value(node: &AxNode) -> Option<String> {
|
||||||
|
node.value
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.value.as_ref())
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map a property name string to the corresponding `AxPropertyName` variant.
|
||||||
|
fn property_by_name(name: &str) -> Option<AxPropertyName> {
|
||||||
|
match name {
|
||||||
|
"focused" => Some(AxPropertyName::Focused),
|
||||||
|
"checked" => Some(AxPropertyName::Checked),
|
||||||
|
"disabled" => Some(AxPropertyName::Disabled),
|
||||||
|
"expanded" => Some(AxPropertyName::Expanded),
|
||||||
|
"required" => Some(AxPropertyName::Required),
|
||||||
|
"selected" => Some(AxPropertyName::Selected),
|
||||||
|
"pressed" => Some(AxPropertyName::Pressed),
|
||||||
|
"readonly" => Some(AxPropertyName::Readonly),
|
||||||
|
"hidden" => Some(AxPropertyName::Hidden),
|
||||||
|
"modal" => Some(AxPropertyName::Modal),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if a node has a boolean property set to true.
|
||||||
|
fn node_has_property(node: &AxNode, prop_name: &str) -> bool {
|
||||||
|
let Some(props) = &node.properties else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let Some(target) = property_by_name(prop_name) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
props.iter().any(|p| {
|
||||||
|
p.name == target
|
||||||
|
&& p.value
|
||||||
|
.value
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| v.as_bool())
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Best-effort CSS selector guess from node attributes.
|
||||||
|
fn guess_selector(node: &AxNode) -> Option<String> {
|
||||||
|
// We don't have DOM attributes directly from the AX tree,
|
||||||
|
// so we can only offer role-based hints. The actual targeting
|
||||||
|
// uses backend_node_id which is precise.
|
||||||
|
let role = node_role(node);
|
||||||
|
let name = node_name(node);
|
||||||
|
|
||||||
|
if name.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build an ARIA selector hint (not used for actual targeting,
|
||||||
|
// just a human-readable hint in debug output).
|
||||||
|
Some(format!(
|
||||||
|
"[role=\"{}\"][name=\"{}\"]",
|
||||||
|
role,
|
||||||
|
truncate_name(&name, 30)
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Truncate a display name to max chars, adding ellipsis if needed.
|
||||||
|
fn truncate_name(s: &str, max: usize) -> String {
|
||||||
|
if s.chars().count() <= max {
|
||||||
|
s.to_string()
|
||||||
|
} else {
|
||||||
|
format!(
|
||||||
|
"{}...",
|
||||||
|
s.chars().take(max.saturating_sub(3)).collect::<String>()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper for formatting a node's display label.
|
||||||
|
struct NodeLabel {
|
||||||
|
role: String,
|
||||||
|
name: String,
|
||||||
|
properties: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for NodeLabel {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
write!(f, "{}", self.role)?;
|
||||||
|
if !self.name.is_empty() {
|
||||||
|
write!(f, " \"{}\"", self.name)?;
|
||||||
|
}
|
||||||
|
if !self.properties.is_empty() {
|
||||||
|
write!(f, " [{}]", self.properties.join(", "))?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::tools::builtin::browser::accessibility::{
|
||||||
|
ElementFilter, ElementRefMap, build_page_repr, truncate_name,
|
||||||
|
};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::accessibility::{
|
||||||
|
AxNode, AxNodeId, AxValue, AxValueType,
|
||||||
|
};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::dom::BackendNodeId;
|
||||||
|
|
||||||
|
fn make_ax_value(s: &str) -> AxValue {
|
||||||
|
let mut v = AxValue::new(AxValueType::String);
|
||||||
|
v.value = Some(serde_json::Value::String(s.to_string()));
|
||||||
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_ax_node(role: &str, name: &str, backend_id: i64) -> AxNode {
|
||||||
|
let mut node = AxNode::new(AxNodeId::from(format!("node_{}", backend_id)), false);
|
||||||
|
node.role = Some(make_ax_value(role));
|
||||||
|
node.name = Some(make_ax_value(name));
|
||||||
|
node.backend_dom_node_id = Some(BackendNodeId::new(backend_id));
|
||||||
|
node
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_page_repr_interactive_filter() {
|
||||||
|
let nodes = vec![
|
||||||
|
make_ax_node("button", "Submit", 1),
|
||||||
|
make_ax_node("link", "Home", 2),
|
||||||
|
make_ax_node("textbox", "Email", 3),
|
||||||
|
make_ax_node("heading", "Welcome", 4), // not interactive
|
||||||
|
make_ax_node("generic", "", 5), // skip role
|
||||||
|
];
|
||||||
|
|
||||||
|
let mut ref_map = ElementRefMap::new();
|
||||||
|
let repr = build_page_repr(
|
||||||
|
"https://example.com",
|
||||||
|
"Test Page",
|
||||||
|
&nodes,
|
||||||
|
ElementFilter::Interactive,
|
||||||
|
&mut ref_map,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(repr.contains("@e1: button \"Submit\""));
|
||||||
|
assert!(repr.contains("@e2: link \"Home\""));
|
||||||
|
assert!(repr.contains("@e3: textbox \"Email\""));
|
||||||
|
assert!(!repr.contains("heading"));
|
||||||
|
assert!(!repr.contains("generic"));
|
||||||
|
assert_eq!(ref_map.len(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_build_page_repr_all_filter() {
|
||||||
|
let nodes = vec![
|
||||||
|
make_ax_node("button", "Submit", 1),
|
||||||
|
make_ax_node("heading", "Welcome", 2),
|
||||||
|
];
|
||||||
|
|
||||||
|
let mut ref_map = ElementRefMap::new();
|
||||||
|
let repr = build_page_repr(
|
||||||
|
"https://example.com",
|
||||||
|
"",
|
||||||
|
&nodes,
|
||||||
|
ElementFilter::All,
|
||||||
|
&mut ref_map,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(repr.contains("button"));
|
||||||
|
assert!(repr.contains("heading"));
|
||||||
|
assert_eq!(ref_map.len(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_element_ref_lookup() {
|
||||||
|
let mut ref_map = ElementRefMap::new();
|
||||||
|
let nodes = vec![make_ax_node("button", "Click me", 1)];
|
||||||
|
build_page_repr(
|
||||||
|
"https://x.com",
|
||||||
|
"",
|
||||||
|
&nodes,
|
||||||
|
ElementFilter::Interactive,
|
||||||
|
&mut ref_map,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(ref_map.get("e1").is_some());
|
||||||
|
assert!(ref_map.get("@e1").is_some()); // with @ prefix
|
||||||
|
assert!(ref_map.get("e99").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_empty_page() {
|
||||||
|
let mut ref_map = ElementRefMap::new();
|
||||||
|
let repr = build_page_repr(
|
||||||
|
"https://empty.com",
|
||||||
|
"",
|
||||||
|
&[],
|
||||||
|
ElementFilter::Interactive,
|
||||||
|
&mut ref_map,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(repr.contains("no interactive elements"));
|
||||||
|
assert!(ref_map.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_truncate_name() {
|
||||||
|
assert_eq!(truncate_name("short", 10), "short");
|
||||||
|
assert_eq!(truncate_name("this is a very long name", 10), "this is...");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,517 @@
|
|||||||
|
//! Headless browser tool for web interaction.
|
||||||
|
//!
|
||||||
|
//! A single `BrowserTool` that dispatches actions via a tagged enum,
|
||||||
|
//! keeping the tool registry clean (one tool, not ten). The LLM sends
|
||||||
|
//! an `action` field to pick the operation:
|
||||||
|
//!
|
||||||
|
//! ```json
|
||||||
|
//! { "action": "navigate", "url": "https://example.com" }
|
||||||
|
//! { "action": "click", "ref": "@e3" }
|
||||||
|
//! { "action": "type", "ref": "@e1", "text": "hello" }
|
||||||
|
//! { "action": "read_page" }
|
||||||
|
//! { "action": "screenshot" }
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! Element references (`@e1`, `@e2`, ...) are assigned by `read_page`
|
||||||
|
//! and remain valid until the next `read_page` call.
|
||||||
|
|
||||||
|
pub mod accessibility;
|
||||||
|
pub mod session;
|
||||||
|
pub mod stealth;
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use serde::Deserialize;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
|
use crate::context::JobContext;
|
||||||
|
use crate::tools::builtin::browser::accessibility::ElementFilter;
|
||||||
|
use crate::tools::builtin::browser::session::BrowserSession;
|
||||||
|
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||||
|
|
||||||
|
/// Actions the LLM can request from the browser tool.
|
||||||
|
///
|
||||||
|
/// Uses serde tagged enum: the JSON `"action"` field selects the variant,
|
||||||
|
/// remaining fields are variant-specific parameters.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
#[serde(tag = "action", rename_all = "snake_case")]
|
||||||
|
enum BrowserAction {
|
||||||
|
/// Navigate to a URL.
|
||||||
|
Navigate { url: String },
|
||||||
|
/// Go back in browser history.
|
||||||
|
Back,
|
||||||
|
/// Go forward in browser history.
|
||||||
|
Forward,
|
||||||
|
/// Read the page's accessibility tree (assigns element refs).
|
||||||
|
ReadPage {
|
||||||
|
/// "interactive" (default) or "all"
|
||||||
|
filter: Option<String>,
|
||||||
|
},
|
||||||
|
/// Click an element by reference ID.
|
||||||
|
Click {
|
||||||
|
/// Element reference like "@e1" or "e1".
|
||||||
|
#[serde(alias = "ref")]
|
||||||
|
ref_id: String,
|
||||||
|
},
|
||||||
|
/// Type text into an element by reference ID.
|
||||||
|
Type {
|
||||||
|
/// Element reference like "@e1" or "e1".
|
||||||
|
#[serde(alias = "ref")]
|
||||||
|
ref_id: String,
|
||||||
|
text: String,
|
||||||
|
},
|
||||||
|
/// Scroll the page.
|
||||||
|
Scroll {
|
||||||
|
/// "up", "down", "left", "right"
|
||||||
|
direction: String,
|
||||||
|
/// Number of scroll steps (default 3).
|
||||||
|
amount: Option<u32>,
|
||||||
|
},
|
||||||
|
/// Capture a screenshot (returns base64 PNG).
|
||||||
|
Screenshot {
|
||||||
|
/// Capture full scrollable page (default false).
|
||||||
|
full_page: Option<bool>,
|
||||||
|
},
|
||||||
|
/// Extract text content from the page or a CSS selector.
|
||||||
|
Extract {
|
||||||
|
/// Optional CSS selector. If omitted, extracts all body text.
|
||||||
|
selector: Option<String>,
|
||||||
|
},
|
||||||
|
/// Wait for a CSS selector to appear or a fixed delay.
|
||||||
|
Wait {
|
||||||
|
/// CSS selector to wait for. If omitted, just sleeps.
|
||||||
|
selector: Option<String>,
|
||||||
|
/// Timeout in milliseconds (default 5000).
|
||||||
|
timeout_ms: Option<u64>,
|
||||||
|
},
|
||||||
|
/// Execute JavaScript (requires user approval).
|
||||||
|
EvalJs { expression: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Headless browser tool for navigating web pages, interacting with
|
||||||
|
/// elements, and extracting content.
|
||||||
|
///
|
||||||
|
/// Uses Chrome/Chromium via the DevTools Protocol. The browser is launched
|
||||||
|
/// lazily on first use and includes basic anti-detection patches.
|
||||||
|
///
|
||||||
|
/// ## Workflow
|
||||||
|
///
|
||||||
|
/// 1. `navigate` to a URL
|
||||||
|
/// 2. `read_page` to get the accessibility tree with element refs
|
||||||
|
/// 3. `click` / `type` using the refs
|
||||||
|
/// 4. `extract` or `screenshot` to get results
|
||||||
|
///
|
||||||
|
/// Element refs (`@e1`, `@e2`) are valid until the next `read_page`.
|
||||||
|
pub struct BrowserTool {
|
||||||
|
/// Lazily initialized browser session. RwLock because `execute` takes `&self`.
|
||||||
|
session: RwLock<Option<BrowserSession>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BrowserTool {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
session: RwLock::new(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ensure the browser session is initialized, launching Chrome if needed.
|
||||||
|
async fn ensure_session(&self) -> Result<(), ToolError> {
|
||||||
|
let needs_launch = self.session.read().await.is_none();
|
||||||
|
if needs_launch {
|
||||||
|
let new_session = BrowserSession::launch().await?;
|
||||||
|
let mut guard = self.session.write().await;
|
||||||
|
if guard.is_none() {
|
||||||
|
*guard = Some(new_session);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for BrowserTool {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl Tool for BrowserTool {
|
||||||
|
fn name(&self) -> &str {
|
||||||
|
"browser"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn description(&self) -> &str {
|
||||||
|
"Control a headless web browser. Navigate pages, read content, click elements, type text, \
|
||||||
|
take screenshots. Use 'read_page' to get an accessibility tree with element references \
|
||||||
|
(@e1, @e2...), then use those refs for 'click' and 'type' actions.\n\n\
|
||||||
|
Actions: navigate, back, forward, read_page, click, type, scroll, screenshot, extract, \
|
||||||
|
wait, eval_js"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parameters_schema(&self) -> serde_json::Value {
|
||||||
|
serde_json::json!({
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"action": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"navigate", "back", "forward", "read_page", "click",
|
||||||
|
"type", "scroll", "screenshot", "extract", "wait", "eval_js"
|
||||||
|
],
|
||||||
|
"description": "The browser action to perform"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "URL to navigate to (for 'navigate' action)"
|
||||||
|
},
|
||||||
|
"ref_id": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Element reference like '@e1' (for 'click' and 'type' actions)"
|
||||||
|
},
|
||||||
|
"text": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Text to type (for 'type' action)"
|
||||||
|
},
|
||||||
|
"direction": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["up", "down", "left", "right"],
|
||||||
|
"description": "Scroll direction (for 'scroll' action)"
|
||||||
|
},
|
||||||
|
"amount": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Scroll steps, default 3 (for 'scroll' action)"
|
||||||
|
},
|
||||||
|
"full_page": {
|
||||||
|
"type": "boolean",
|
||||||
|
"description": "Capture full scrollable page (for 'screenshot' action)"
|
||||||
|
},
|
||||||
|
"selector": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "CSS selector (for 'extract' and 'wait' actions)"
|
||||||
|
},
|
||||||
|
"timeout_ms": {
|
||||||
|
"type": "integer",
|
||||||
|
"description": "Timeout in milliseconds (for 'wait' action, default 5000)"
|
||||||
|
},
|
||||||
|
"filter": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": ["interactive", "all"],
|
||||||
|
"description": "Element filter for 'read_page' (default: interactive)"
|
||||||
|
},
|
||||||
|
"expression": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "JavaScript expression (for 'eval_js' action)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["action"]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute(
|
||||||
|
&self,
|
||||||
|
params: serde_json::Value,
|
||||||
|
_ctx: &JobContext,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
|
let action: BrowserAction = serde_json::from_value(params)
|
||||||
|
.map_err(|e| ToolError::InvalidParameters(format!("Invalid browser action: {}", e)))?;
|
||||||
|
|
||||||
|
// Launch browser on first use.
|
||||||
|
self.ensure_session().await?;
|
||||||
|
|
||||||
|
match action {
|
||||||
|
BrowserAction::Navigate { url } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let title = session.navigate(&url).await?;
|
||||||
|
let current_url = session.current_url().await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({
|
||||||
|
"url": current_url,
|
||||||
|
"title": title,
|
||||||
|
"status": "navigated"
|
||||||
|
}),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Back => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
session.go_back().await?;
|
||||||
|
let url = session.current_url().await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({ "url": url, "status": "navigated_back" }),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Forward => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
session.go_forward().await?;
|
||||||
|
let url = session.current_url().await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({ "url": url, "status": "navigated_forward" }),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::ReadPage { filter } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let element_filter = ElementFilter::from_str_opt(filter.as_deref());
|
||||||
|
let repr = session.read_page(element_filter).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::text(repr, start.elapsed()))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Click { ref_id } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
session.click_element(&ref_id).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({ "status": "clicked", "ref": ref_id }),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Type { ref_id, text } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
session.type_text(&ref_id, &text).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({
|
||||||
|
"status": "typed",
|
||||||
|
"ref": ref_id,
|
||||||
|
"length": text.len()
|
||||||
|
}),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Scroll { direction, amount } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let steps = amount.unwrap_or(3);
|
||||||
|
session.scroll(&direction, steps).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({
|
||||||
|
"status": "scrolled",
|
||||||
|
"direction": direction,
|
||||||
|
"amount": steps
|
||||||
|
}),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Screenshot { full_page } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let b64 = session.screenshot(full_page.unwrap_or(false)).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({
|
||||||
|
"format": "png",
|
||||||
|
"encoding": "base64",
|
||||||
|
"data": b64,
|
||||||
|
"full_page": full_page.unwrap_or(false)
|
||||||
|
}),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Extract { selector } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let text = session.extract_text(selector.as_deref()).await?;
|
||||||
|
|
||||||
|
// Truncate very long text to avoid blowing up context.
|
||||||
|
let truncated = if text.len() > 32_000 {
|
||||||
|
format!(
|
||||||
|
"{}...\n\n[truncated, {} total chars]",
|
||||||
|
&text[..32_000],
|
||||||
|
text.len()
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
text.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(ToolOutput::text(&truncated, start.elapsed()).with_raw(text))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::Wait {
|
||||||
|
selector,
|
||||||
|
timeout_ms,
|
||||||
|
} => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let timeout = timeout_ms.unwrap_or(5000);
|
||||||
|
let found = session.wait(selector.as_deref(), timeout).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({
|
||||||
|
"found": found,
|
||||||
|
"selector": selector,
|
||||||
|
"timeout_ms": timeout
|
||||||
|
}),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
BrowserAction::EvalJs { expression } => {
|
||||||
|
let session = self.session.read().await;
|
||||||
|
let session = session.as_ref().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed("Browser session not initialized".to_string())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let result = session.eval_js(&expression).await?;
|
||||||
|
|
||||||
|
Ok(ToolOutput::success(
|
||||||
|
serde_json::json!({ "result": result }),
|
||||||
|
start.elapsed(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn estimated_duration(&self, _params: &serde_json::Value) -> Option<Duration> {
|
||||||
|
Some(Duration::from_secs(10))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn requires_sanitization(&self) -> bool {
|
||||||
|
true // Page content is untrusted external data
|
||||||
|
}
|
||||||
|
|
||||||
|
fn requires_approval(&self) -> bool {
|
||||||
|
true // Browser navigates to external sites, executes JS
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::tools::builtin::browser::BrowserTool;
|
||||||
|
use crate::tools::tool::Tool;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_browser_tool_metadata() {
|
||||||
|
let tool = BrowserTool::new();
|
||||||
|
assert_eq!(tool.name(), "browser");
|
||||||
|
assert!(tool.requires_approval());
|
||||||
|
assert!(tool.requires_sanitization());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_schema_has_action_enum() {
|
||||||
|
let tool = BrowserTool::new();
|
||||||
|
let schema = tool.parameters_schema();
|
||||||
|
|
||||||
|
let action_prop = schema.get("properties").and_then(|p| p.get("action"));
|
||||||
|
assert!(action_prop.is_some());
|
||||||
|
|
||||||
|
let action_enum = action_prop.and_then(|a| a.get("enum"));
|
||||||
|
assert!(action_enum.is_some());
|
||||||
|
|
||||||
|
let actions: Vec<&str> = action_enum
|
||||||
|
.and_then(|e| e.as_array())
|
||||||
|
.map(|arr| arr.iter().filter_map(|v| v.as_str()).collect())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
assert!(actions.contains(&"navigate"));
|
||||||
|
assert!(actions.contains(&"click"));
|
||||||
|
assert!(actions.contains(&"type"));
|
||||||
|
assert!(actions.contains(&"read_page"));
|
||||||
|
assert!(actions.contains(&"screenshot"));
|
||||||
|
assert!(actions.contains(&"eval_js"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_action_deserialization() {
|
||||||
|
use super::BrowserAction;
|
||||||
|
|
||||||
|
// Navigate
|
||||||
|
let action: BrowserAction = serde_json::from_value(
|
||||||
|
serde_json::json!({"action": "navigate", "url": "https://x.com"}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(action, BrowserAction::Navigate { url } if url == "https://x.com"));
|
||||||
|
|
||||||
|
// Click with "ref" alias
|
||||||
|
let action: BrowserAction =
|
||||||
|
serde_json::from_value(serde_json::json!({"action": "click", "ref": "@e1"})).unwrap();
|
||||||
|
assert!(matches!(action, BrowserAction::Click { ref_id } if ref_id == "@e1"));
|
||||||
|
|
||||||
|
// Click with "ref_id"
|
||||||
|
let action: BrowserAction =
|
||||||
|
serde_json::from_value(serde_json::json!({"action": "click", "ref_id": "e2"})).unwrap();
|
||||||
|
assert!(matches!(action, BrowserAction::Click { ref_id } if ref_id == "e2"));
|
||||||
|
|
||||||
|
// Type
|
||||||
|
let action: BrowserAction = serde_json::from_value(
|
||||||
|
serde_json::json!({"action": "type", "ref": "@e1", "text": "hello"}),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
matches!(action, BrowserAction::Type { ref_id, text } if ref_id == "@e1" && text == "hello")
|
||||||
|
);
|
||||||
|
|
||||||
|
// ReadPage with default filter
|
||||||
|
let action: BrowserAction =
|
||||||
|
serde_json::from_value(serde_json::json!({"action": "read_page"})).unwrap();
|
||||||
|
assert!(matches!(action, BrowserAction::ReadPage { filter: None }));
|
||||||
|
|
||||||
|
// Screenshot
|
||||||
|
let action: BrowserAction =
|
||||||
|
serde_json::from_value(serde_json::json!({"action": "screenshot", "full_page": true}))
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
action,
|
||||||
|
BrowserAction::Screenshot {
|
||||||
|
full_page: Some(true)
|
||||||
|
}
|
||||||
|
));
|
||||||
|
|
||||||
|
// Invalid action
|
||||||
|
let result: Result<BrowserAction, _> =
|
||||||
|
serde_json::from_value(serde_json::json!({"action": "fly_to_moon"}));
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,587 @@
|
|||||||
|
//! Browser session management.
|
||||||
|
//!
|
||||||
|
//! Owns the Chrome process lifecycle and per-tab state. Sessions are spawned
|
||||||
|
//! lazily on first browser action and torn down when dropped.
|
||||||
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! BrowserSession
|
||||||
|
//! ├── Browser (chromiumoxide, owns Chrome child process)
|
||||||
|
//! ├── handler_task (JoinHandle polling CDP WebSocket)
|
||||||
|
//! ├── tabs: HashMap<tab_id, Page>
|
||||||
|
//! ├── active_tab: current tab id
|
||||||
|
//! └── element_refs: ElementRefMap (valid until next read_page)
|
||||||
|
//! ```
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use chromiumoxide::Page;
|
||||||
|
use chromiumoxide::browser::{Browser, BrowserConfig};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::accessibility::GetFullAxTreeParams;
|
||||||
|
use chromiumoxide::cdp::browser_protocol::dom::{GetBoxModelParams, ScrollIntoViewIfNeededParams};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::input::{
|
||||||
|
DispatchMouseEventParams, DispatchMouseEventType, InsertTextParams, MouseButton,
|
||||||
|
};
|
||||||
|
use chromiumoxide::cdp::browser_protocol::page::CaptureScreenshotFormat;
|
||||||
|
use chromiumoxide::page::ScreenshotParams;
|
||||||
|
use futures::StreamExt;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
|
use crate::tools::builtin::browser::accessibility::{
|
||||||
|
ElementFilter, ElementRefMap, build_page_repr,
|
||||||
|
};
|
||||||
|
use crate::tools::builtin::browser::stealth;
|
||||||
|
use crate::tools::tool::ToolError;
|
||||||
|
|
||||||
|
/// Manages a Chrome browser instance and its tabs.
|
||||||
|
pub struct BrowserSession {
|
||||||
|
#[allow(dead_code)] // Used by new_tab() which is reserved for tab management actions
|
||||||
|
browser: Browser,
|
||||||
|
_handler_task: JoinHandle<()>,
|
||||||
|
tabs: HashMap<String, Page>,
|
||||||
|
active_tab: String,
|
||||||
|
element_refs: Arc<RwLock<ElementRefMap>>,
|
||||||
|
#[allow(dead_code)] // Used by new_tab() which is reserved for tab management actions
|
||||||
|
stealth_js: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BrowserSession {
|
||||||
|
/// Launch a new Chrome browser session.
|
||||||
|
///
|
||||||
|
/// Locates Chrome on the system, applies stealth patches, and opens
|
||||||
|
/// an initial blank tab.
|
||||||
|
pub async fn launch() -> Result<Self, ToolError> {
|
||||||
|
let chrome_path = find_chrome().ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed(
|
||||||
|
"Chrome/Chromium not found. Install Chrome or set CHROME_PATH.".to_string(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Shared profile so the agent accumulates useful state across sessions
|
||||||
|
// (logged-in sessions, dismissed cookie banners, local storage).
|
||||||
|
// Delete ~/.ironclaw/browser/profile/ to reset.
|
||||||
|
let profile_dir = browser_profile_dir();
|
||||||
|
std::fs::create_dir_all(&profile_dir).map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to create browser profile dir: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut config_builder = BrowserConfig::builder()
|
||||||
|
.chrome_executable(&chrome_path)
|
||||||
|
.user_data_dir(&profile_dir)
|
||||||
|
.window_size(1920, 1080)
|
||||||
|
.no_sandbox();
|
||||||
|
|
||||||
|
for arg in stealth::stealth_args() {
|
||||||
|
config_builder = config_builder.arg(arg);
|
||||||
|
}
|
||||||
|
|
||||||
|
let config = config_builder.build().map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to build browser config: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let (browser, mut handler) = Browser::launch(config)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to launch Chrome: {}", e)))?;
|
||||||
|
|
||||||
|
// The handler must be polled continuously or the CDP connection dies.
|
||||||
|
let handler_task = tokio::spawn(async move {
|
||||||
|
while let Some(event) = handler.next().await {
|
||||||
|
if event.is_err() {
|
||||||
|
tracing::warn!("Browser handler error: {:?}", event);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Open initial tab.
|
||||||
|
let page = browser.new_page("about:blank").await.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to open initial tab: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Inject stealth JS on every new document load for this page.
|
||||||
|
let stealth_js = stealth::stealth_js().to_string();
|
||||||
|
page.evaluate_on_new_document(stealth_js.clone())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to inject stealth JS: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let tab_id = "tab0".to_string();
|
||||||
|
let mut tabs = HashMap::new();
|
||||||
|
tabs.insert(tab_id.clone(), page);
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
browser,
|
||||||
|
_handler_task: handler_task,
|
||||||
|
tabs,
|
||||||
|
active_tab: tab_id,
|
||||||
|
element_refs: Arc::new(RwLock::new(ElementRefMap::new())),
|
||||||
|
stealth_js,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get the active page, or error if session is broken.
|
||||||
|
fn active_page(&self) -> Result<&Page, ToolError> {
|
||||||
|
self.tabs.get(&self.active_tab).ok_or_else(|| {
|
||||||
|
ToolError::ExecutionFailed(format!("No active tab: {}", self.active_tab))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Navigation ---
|
||||||
|
|
||||||
|
pub async fn navigate(&self, url: &str) -> Result<String, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
page.goto(url)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExternalService(format!("Navigation failed: {}", e)))?;
|
||||||
|
|
||||||
|
let title = page
|
||||||
|
.get_title()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to get page title: {}", e)))?
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
Ok(title)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn go_back(&self) -> Result<(), ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
page.evaluate("window.history.back()")
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to go back: {}", e)))?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn go_forward(&self) -> Result<(), ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
page.evaluate("window.history.forward()")
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to go forward: {}", e)))?;
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Page reading ---
|
||||||
|
|
||||||
|
/// Build accessibility tree representation and update element refs.
|
||||||
|
pub async fn read_page(&self, filter: ElementFilter) -> Result<String, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
let url = page
|
||||||
|
.url()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to get URL: {}", e)))?
|
||||||
|
.unwrap_or_else(|| "about:blank".to_string());
|
||||||
|
|
||||||
|
let title = page
|
||||||
|
.get_title()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to get title: {}", e)))?
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Fetch full accessibility tree via CDP.
|
||||||
|
let ax_result = page
|
||||||
|
.execute(GetFullAxTreeParams::default())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to get accessibility tree: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let nodes = ax_result.result.nodes;
|
||||||
|
|
||||||
|
let mut ref_map = self.element_refs.write().await;
|
||||||
|
let repr = build_page_repr(&url, &title, &nodes, filter, &mut ref_map);
|
||||||
|
|
||||||
|
Ok(repr)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract text content from the page or a CSS selector.
|
||||||
|
pub async fn extract_text(&self, selector: Option<&str>) -> Result<String, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
let js = match selector {
|
||||||
|
Some(sel) => {
|
||||||
|
let escaped = serde_json::to_string(sel).map_err(|e| {
|
||||||
|
ToolError::InvalidParameters(format!("Invalid selector: {}", e))
|
||||||
|
})?;
|
||||||
|
format!(
|
||||||
|
"(() => {{ const el = document.querySelector({}); return el ? el.innerText : null; }})()",
|
||||||
|
escaped
|
||||||
|
)
|
||||||
|
}
|
||||||
|
None => "document.body.innerText".to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let result: Option<String> = page
|
||||||
|
.evaluate(js.as_str())
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to extract text: {}", e)))?
|
||||||
|
.into_value()
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to deserialize text: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(result.unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Interaction ---
|
||||||
|
|
||||||
|
/// Click an element by reference ID (e.g., "e1" or "@e1").
|
||||||
|
///
|
||||||
|
/// Uses DOM.scrollIntoViewIfNeeded + DOM.getBoxModel to find the element's
|
||||||
|
/// center coordinates, then dispatches mouse press + release at that point.
|
||||||
|
pub async fn click_element(&self, ref_id: &str) -> Result<(), ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
let refs = self.element_refs.read().await;
|
||||||
|
|
||||||
|
let elem_ref = refs.get(ref_id).ok_or_else(|| {
|
||||||
|
ToolError::InvalidParameters(format!(
|
||||||
|
"Unknown element reference '{}'. Call browser with action 'read_page' first.",
|
||||||
|
ref_id
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let backend_node_id = elem_ref.backend_node_id;
|
||||||
|
drop(refs);
|
||||||
|
|
||||||
|
// Scroll the element into the viewport.
|
||||||
|
page.execute(
|
||||||
|
ScrollIntoViewIfNeededParams::builder()
|
||||||
|
.backend_node_id(backend_node_id)
|
||||||
|
.build(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to scroll element into view: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Get element's bounding box via DOM.getBoxModel.
|
||||||
|
let box_result = page
|
||||||
|
.execute(
|
||||||
|
GetBoxModelParams::builder()
|
||||||
|
.backend_node_id(backend_node_id)
|
||||||
|
.build(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to get element box model: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Content quad is [x1,y1, x2,y2, x3,y3, x4,y4]. Center = average of 4 corners.
|
||||||
|
let content = box_result.result.model.content.inner();
|
||||||
|
if content.len() < 8 {
|
||||||
|
return Err(ToolError::ExecutionFailed(
|
||||||
|
"Element has no valid bounding box".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let x = (content[0] + content[2] + content[4] + content[6]) / 4.0;
|
||||||
|
let y = (content[1] + content[3] + content[5] + content[7]) / 4.0;
|
||||||
|
|
||||||
|
// Dispatch mouse press + release at center of element.
|
||||||
|
page.execute(
|
||||||
|
DispatchMouseEventParams::builder()
|
||||||
|
.r#type(DispatchMouseEventType::MousePressed)
|
||||||
|
.x(x)
|
||||||
|
.y(y)
|
||||||
|
.button(MouseButton::Left)
|
||||||
|
.click_count(1)
|
||||||
|
.build()
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to build mouse event: {}", e))
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Mouse press failed: {}", e)))?;
|
||||||
|
|
||||||
|
page.execute(
|
||||||
|
DispatchMouseEventParams::builder()
|
||||||
|
.r#type(DispatchMouseEventType::MouseReleased)
|
||||||
|
.x(x)
|
||||||
|
.y(y)
|
||||||
|
.button(MouseButton::Left)
|
||||||
|
.click_count(1)
|
||||||
|
.build()
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to build mouse event: {}", e))
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Mouse release failed: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Type text into an element by reference ID.
|
||||||
|
pub async fn type_text(&self, ref_id: &str, text: &str) -> Result<(), ToolError> {
|
||||||
|
// First click to focus the element.
|
||||||
|
self.click_element(ref_id).await?;
|
||||||
|
|
||||||
|
// Brief delay to let focus settle.
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
|
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
// Use CDP insertText for reliable IME-style text entry.
|
||||||
|
page.execute(InsertTextParams::new(text))
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to type text: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scroll the page.
|
||||||
|
pub async fn scroll(&self, direction: &str, amount: u32) -> Result<(), ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
let (dx, dy) = match direction {
|
||||||
|
"up" => (0, -(amount as i32 * 100)),
|
||||||
|
"down" => (0, amount as i32 * 100),
|
||||||
|
"left" => (-(amount as i32 * 100), 0),
|
||||||
|
"right" => (amount as i32 * 100, 0),
|
||||||
|
_ => {
|
||||||
|
return Err(ToolError::InvalidParameters(format!(
|
||||||
|
"Invalid scroll direction '{}'. Use: up, down, left, right",
|
||||||
|
direction
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let js = format!("window.scrollBy({}, {})", dx, dy);
|
||||||
|
page.evaluate(js.as_str())
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Scroll failed: {}", e)))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait for a CSS selector to appear, or a fixed timeout.
|
||||||
|
pub async fn wait(&self, selector: Option<&str>, timeout_ms: u64) -> Result<bool, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
let timeout = std::time::Duration::from_millis(timeout_ms);
|
||||||
|
|
||||||
|
match selector {
|
||||||
|
Some(sel) => {
|
||||||
|
let poll_interval = std::time::Duration::from_millis(100);
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let escaped = serde_json::to_string(sel).map_err(|e| {
|
||||||
|
ToolError::InvalidParameters(format!("Invalid selector: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let js = format!("!!document.querySelector({})", escaped);
|
||||||
|
let found: bool = page
|
||||||
|
.evaluate(js.as_str())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Wait poll failed: {}", e))
|
||||||
|
})?
|
||||||
|
.into_value()
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if found {
|
||||||
|
return Ok(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
if start.elapsed() >= timeout {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
tokio::time::sleep(poll_interval).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
tokio::time::sleep(timeout).await;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Screenshots ---
|
||||||
|
|
||||||
|
/// Capture a screenshot as base64-encoded PNG.
|
||||||
|
pub async fn screenshot(&self, full_page: bool) -> Result<String, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
let params = ScreenshotParams::builder()
|
||||||
|
.format(CaptureScreenshotFormat::Png)
|
||||||
|
.full_page(full_page)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
let bytes = page
|
||||||
|
.screenshot(params)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Screenshot failed: {}", e)))?;
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
Ok(base64::engine::general_purpose::STANDARD.encode(&bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- JavaScript ---
|
||||||
|
|
||||||
|
/// Execute arbitrary JavaScript and return the result.
|
||||||
|
pub async fn eval_js(&self, expression: &str) -> Result<serde_json::Value, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
|
||||||
|
let result = page
|
||||||
|
.evaluate(expression)
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("JS evaluation failed: {}", e)))?;
|
||||||
|
|
||||||
|
let value: serde_json::Value = result.into_value().unwrap_or(serde_json::Value::Null);
|
||||||
|
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Tab management ---
|
||||||
|
|
||||||
|
/// Open a new tab and make it active.
|
||||||
|
#[allow(dead_code)] // Reserved for tab management actions
|
||||||
|
pub async fn new_tab(&mut self, url: &str) -> Result<String, ToolError> {
|
||||||
|
let page =
|
||||||
|
self.browser.new_page(url).await.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to open new tab: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Inject stealth JS on the new page too.
|
||||||
|
page.evaluate_on_new_document(self.stealth_js.clone())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
ToolError::ExecutionFailed(format!("Failed to inject stealth JS on new tab: {}", e))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let tab_id = format!("tab{}", self.tabs.len());
|
||||||
|
self.tabs.insert(tab_id.clone(), page);
|
||||||
|
self.active_tab = tab_id.clone();
|
||||||
|
|
||||||
|
// Clear element refs since we're on a new page.
|
||||||
|
self.element_refs.write().await.reset();
|
||||||
|
|
||||||
|
Ok(tab_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// List open tabs.
|
||||||
|
#[allow(dead_code)] // Reserved for tab management actions
|
||||||
|
pub fn list_tabs(&self) -> Vec<String> {
|
||||||
|
self.tabs.keys().cloned().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Switch to a different tab.
|
||||||
|
#[allow(dead_code)] // Reserved for tab management actions
|
||||||
|
pub async fn switch_tab(&mut self, tab_id: &str) -> Result<(), ToolError> {
|
||||||
|
if !self.tabs.contains_key(tab_id) {
|
||||||
|
return Err(ToolError::InvalidParameters(format!(
|
||||||
|
"Unknown tab '{}'. Open tabs: {:?}",
|
||||||
|
tab_id,
|
||||||
|
self.list_tabs()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
self.active_tab = tab_id.to_string();
|
||||||
|
// Clear element refs when switching tabs.
|
||||||
|
self.element_refs.write().await.reset();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Get current page URL.
|
||||||
|
pub async fn current_url(&self) -> Result<String, ToolError> {
|
||||||
|
let page = self.active_page()?;
|
||||||
|
page.url()
|
||||||
|
.await
|
||||||
|
.map_err(|e| ToolError::ExecutionFailed(format!("Failed to get URL: {}", e)))
|
||||||
|
.map(|u| u.unwrap_or_else(|| "about:blank".to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for BrowserSession {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
tracing::debug!("Browser session dropping, Chrome process will be cleaned up");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns `~/.ironclaw/browser/profile/`.
|
||||||
|
fn browser_profile_dir() -> PathBuf {
|
||||||
|
dirs::home_dir()
|
||||||
|
.unwrap_or_else(|| PathBuf::from("."))
|
||||||
|
.join(".ironclaw")
|
||||||
|
.join("browser")
|
||||||
|
.join("profile")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Search common locations for a Chrome/Chromium binary.
|
||||||
|
pub fn find_chrome() -> Option<PathBuf> {
|
||||||
|
// Environment variable override.
|
||||||
|
if let Ok(path) = std::env::var("CHROME_PATH") {
|
||||||
|
let p = PathBuf::from(&path);
|
||||||
|
if p.exists() {
|
||||||
|
return Some(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let candidates = if cfg!(target_os = "macos") {
|
||||||
|
vec![
|
||||||
|
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||||
|
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||||
|
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||||
|
"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
|
||||||
|
]
|
||||||
|
} else if cfg!(target_os = "linux") {
|
||||||
|
vec![
|
||||||
|
"/usr/bin/google-chrome",
|
||||||
|
"/usr/bin/google-chrome-stable",
|
||||||
|
"/usr/bin/chromium",
|
||||||
|
"/usr/bin/chromium-browser",
|
||||||
|
"/snap/bin/chromium",
|
||||||
|
]
|
||||||
|
} else {
|
||||||
|
// Windows paths.
|
||||||
|
vec![
|
||||||
|
r"C:\Program Files\Google\Chrome\Application\chrome.exe",
|
||||||
|
r"C:\Program Files (x86)\Google\Chrome\Application\chrome.exe",
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
|
for candidate in candidates {
|
||||||
|
let p = PathBuf::from(candidate);
|
||||||
|
if p.exists() {
|
||||||
|
return Some(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
which_chrome_in_path()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check if chrome/chromium is available in PATH.
|
||||||
|
fn which_chrome_in_path() -> Option<PathBuf> {
|
||||||
|
let path_var = std::env::var("PATH").ok()?;
|
||||||
|
let separator = if cfg!(windows) { ';' } else { ':' };
|
||||||
|
for name in &["google-chrome", "chromium", "chromium-browser", "chrome"] {
|
||||||
|
for dir in path_var.split(separator) {
|
||||||
|
let candidate = PathBuf::from(dir).join(name);
|
||||||
|
if candidate.exists() {
|
||||||
|
return Some(candidate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::tools::builtin::browser::session::find_chrome;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_find_chrome_returns_path_or_none() {
|
||||||
|
let result = find_chrome();
|
||||||
|
if let Some(path) = &result {
|
||||||
|
assert!(
|
||||||
|
path.exists(),
|
||||||
|
"find_chrome returned non-existent path: {:?}",
|
||||||
|
path
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
//! Anti-detection JavaScript patches for headless Chrome.
|
||||||
|
//!
|
||||||
|
//! Injects scripts via `Page.addScriptToEvaluateOnNewDocument` to suppress
|
||||||
|
//! common bot-detection signals. Handles ~80% of detection for legitimate
|
||||||
|
//! browsing (not adversarial scraping against Cloudflare Enterprise).
|
||||||
|
//!
|
||||||
|
//! What we patch:
|
||||||
|
//! - `navigator.webdriver` (trivial but still checked)
|
||||||
|
//! - `navigator.plugins` (headless has empty plugin list)
|
||||||
|
//! - `navigator.languages` (match system locale)
|
||||||
|
//! - `chrome.runtime` (looks like a real extension API)
|
||||||
|
//! - `HeadlessChrome` user-agent substring (suppressed via launch flags)
|
||||||
|
|
||||||
|
/// Chrome launch arguments that reduce detection surface.
|
||||||
|
pub fn stealth_args() -> Vec<&'static str> {
|
||||||
|
vec![
|
||||||
|
"--disable-blink-features=AutomationControlled",
|
||||||
|
"--no-first-run",
|
||||||
|
"--no-default-browser-check",
|
||||||
|
"--disable-infobars",
|
||||||
|
"--disable-background-networking",
|
||||||
|
"--disable-prompt-on-repost",
|
||||||
|
"--disable-hang-monitor",
|
||||||
|
"--disable-sync",
|
||||||
|
"--metrics-recording-only",
|
||||||
|
"--no-service-autorun",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JavaScript injected before any page scripts run.
|
||||||
|
///
|
||||||
|
/// This covers the most common fingerprinting checks. Each patch is
|
||||||
|
/// a self-contained IIFE so failures in one don't break the others.
|
||||||
|
pub fn stealth_js() -> &'static str {
|
||||||
|
r#"
|
||||||
|
// --- navigator.webdriver ---
|
||||||
|
// CDP sets this to true; real browsers have it undefined or false.
|
||||||
|
(() => {
|
||||||
|
Object.defineProperty(navigator, 'webdriver', {
|
||||||
|
get: () => undefined,
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// --- navigator.plugins ---
|
||||||
|
// Headless Chrome reports an empty plugin array. Real Chrome on desktop
|
||||||
|
// always has at least these two. We fake the array shape.
|
||||||
|
(() => {
|
||||||
|
const pluginData = [
|
||||||
|
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer',
|
||||||
|
description: 'Portable Document Format' },
|
||||||
|
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
|
||||||
|
description: '' },
|
||||||
|
];
|
||||||
|
|
||||||
|
const makeMimeType = (type_, suffixes, desc, plugin) => {
|
||||||
|
const mt = Object.create(MimeType.prototype);
|
||||||
|
Object.defineProperties(mt, {
|
||||||
|
type: { get: () => type_ },
|
||||||
|
suffixes: { get: () => suffixes },
|
||||||
|
description: { get: () => desc },
|
||||||
|
enabledPlugin: { get: () => plugin },
|
||||||
|
});
|
||||||
|
return mt;
|
||||||
|
};
|
||||||
|
|
||||||
|
const makePlugin = (data) => {
|
||||||
|
const p = Object.create(Plugin.prototype);
|
||||||
|
const mimes = [makeMimeType('application/pdf', 'pdf', 'Portable Document Format', p)];
|
||||||
|
Object.defineProperties(p, {
|
||||||
|
name: { get: () => data.name },
|
||||||
|
filename: { get: () => data.filename },
|
||||||
|
description: { get: () => data.description },
|
||||||
|
length: { get: () => mimes.length },
|
||||||
|
0: { get: () => mimes[0] },
|
||||||
|
});
|
||||||
|
p.item = (i) => mimes[i] || null;
|
||||||
|
p.namedItem = (name) => mimes.find(m => m.type === name) || null;
|
||||||
|
return p;
|
||||||
|
};
|
||||||
|
|
||||||
|
const plugins = pluginData.map(makePlugin);
|
||||||
|
const pluginArray = Object.create(PluginArray.prototype);
|
||||||
|
Object.defineProperties(pluginArray, {
|
||||||
|
length: { get: () => plugins.length },
|
||||||
|
0: { get: () => plugins[0] },
|
||||||
|
1: { get: () => plugins[1] },
|
||||||
|
});
|
||||||
|
pluginArray.item = (i) => plugins[i] || null;
|
||||||
|
pluginArray.namedItem = (name) => plugins.find(p => p.name === name) || null;
|
||||||
|
pluginArray.refresh = () => {};
|
||||||
|
pluginArray[Symbol.iterator] = function* () { yield* plugins; };
|
||||||
|
|
||||||
|
Object.defineProperty(navigator, 'plugins', {
|
||||||
|
get: () => pluginArray,
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// --- navigator.languages ---
|
||||||
|
// Headless sometimes reports just ['en'] instead of a realistic list.
|
||||||
|
(() => {
|
||||||
|
Object.defineProperty(navigator, 'languages', {
|
||||||
|
get: () => ['en-US', 'en'],
|
||||||
|
configurable: true,
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
|
// --- chrome.runtime ---
|
||||||
|
// Bot detectors check for chrome.runtime to see if it's a real Chrome
|
||||||
|
// extension environment. CDP-controlled Chrome has a broken stub.
|
||||||
|
(() => {
|
||||||
|
if (!window.chrome) window.chrome = {};
|
||||||
|
if (!window.chrome.runtime) {
|
||||||
|
window.chrome.runtime = {
|
||||||
|
connect: () => {},
|
||||||
|
sendMessage: () => {},
|
||||||
|
id: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
// --- Permissions API ---
|
||||||
|
// Headless reports 'denied' for notification permissions by default,
|
||||||
|
// which is a known fingerprinting signal.
|
||||||
|
(() => {
|
||||||
|
const originalQuery = window.Permissions?.prototype?.query;
|
||||||
|
if (originalQuery) {
|
||||||
|
window.Permissions.prototype.query = function(params) {
|
||||||
|
if (params?.name === 'notifications') {
|
||||||
|
return Promise.resolve({ state: 'prompt', onchange: null });
|
||||||
|
}
|
||||||
|
return originalQuery.call(this, params);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
"#
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::tools::builtin::browser::stealth;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stealth_js_is_not_empty() {
|
||||||
|
let js = stealth::stealth_js();
|
||||||
|
assert!(js.len() > 100);
|
||||||
|
assert!(js.contains("navigator"));
|
||||||
|
assert!(js.contains("webdriver"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn stealth_args_are_valid_flags() {
|
||||||
|
for arg in stealth::stealth_args() {
|
||||||
|
assert!(arg.starts_with("--"), "arg should start with --: {}", arg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
/// Simple echo tool for testing.
|
/// Simple echo tool for testing.
|
||||||
pub struct EchoTool;
|
pub struct EchoTool;
|
||||||
@@ -38,12 +38,7 @@ impl Tool for EchoTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let message = params
|
let message = require_str(¶ms, "message")?;
|
||||||
.get("message")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'message' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(ToolOutput::text(message, start.elapsed()))
|
Ok(ToolOutput::text(message, start.elapsed()))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,136 +0,0 @@
|
|||||||
//! E-commerce tool for shopping and price comparison.
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
|
|
||||||
use crate::context::JobContext;
|
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
|
||||||
|
|
||||||
/// Tool for e-commerce operations (Amazon, price comparison, etc.).
|
|
||||||
pub struct EcommerceTool {
|
|
||||||
// TODO: Add API clients
|
|
||||||
}
|
|
||||||
|
|
||||||
impl EcommerceTool {
|
|
||||||
/// Create a new e-commerce tool.
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for EcommerceTool {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Tool for EcommerceTool {
|
|
||||||
fn name(&self) -> &str {
|
|
||||||
"ecommerce"
|
|
||||||
}
|
|
||||||
|
|
||||||
fn description(&self) -> &str {
|
|
||||||
"Search products, compare prices, and find deals across e-commerce platforms."
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parameters_schema(&self) -> serde_json::Value {
|
|
||||||
serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"action": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["search", "get_product", "compare_prices", "track_price"],
|
|
||||||
"description": "The e-commerce action to perform"
|
|
||||||
},
|
|
||||||
"query": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Search query (for search action)"
|
|
||||||
},
|
|
||||||
"product_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Product ID or ASIN (for get_product, compare_prices)"
|
|
||||||
},
|
|
||||||
"platform": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["amazon", "ebay", "walmart", "all"],
|
|
||||||
"description": "E-commerce platform to search"
|
|
||||||
},
|
|
||||||
"max_price": {
|
|
||||||
"type": "number",
|
|
||||||
"description": "Maximum price filter"
|
|
||||||
},
|
|
||||||
"category": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Product category filter"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["action"]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
params: serde_json::Value,
|
|
||||||
_ctx: &JobContext,
|
|
||||||
) -> Result<ToolOutput, ToolError> {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
|
|
||||||
let action = params
|
|
||||||
.get("action")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'action' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// TODO: Implement actual e-commerce API integrations
|
|
||||||
let result = match action {
|
|
||||||
"search" => {
|
|
||||||
let query = params.get("query").and_then(|v| v.as_str()).unwrap_or("");
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"query": query,
|
|
||||||
"results": [],
|
|
||||||
"message": "E-commerce integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_product" => {
|
|
||||||
let product_id = params
|
|
||||||
.get("product_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'product_id' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"product_id": product_id,
|
|
||||||
"found": false,
|
|
||||||
"message": "E-commerce integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"compare_prices" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"prices": [],
|
|
||||||
"message": "E-commerce integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"track_price" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"tracking": false,
|
|
||||||
"message": "E-commerce integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(ToolError::InvalidParameters(format!(
|
|
||||||
"unknown action: {}",
|
|
||||||
action
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(ToolOutput::success(result, start.elapsed()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn requires_sanitization(&self) -> bool {
|
|
||||||
true // External e-commerce data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,7 +9,7 @@ use async_trait::async_trait;
|
|||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::extensions::{ExtensionKind, ExtensionManager};
|
use crate::extensions::{ExtensionKind, ExtensionManager};
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
// ── tool_search ──────────────────────────────────────────────────────────
|
// ── tool_search ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -133,10 +133,7 @@ impl Tool for ToolInstallTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("name is required".to_string()))?;
|
|
||||||
|
|
||||||
let url = params.get("url").and_then(|v| v.as_str());
|
let url = params.get("url").and_then(|v| v.as_str());
|
||||||
|
|
||||||
@@ -210,10 +207,7 @@ impl Tool for ToolAuthTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("name is required".to_string()))?;
|
|
||||||
|
|
||||||
let result = self
|
let result = self
|
||||||
.manager
|
.manager
|
||||||
@@ -306,10 +300,7 @@ impl Tool for ToolActivateTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("name is required".to_string()))?;
|
|
||||||
|
|
||||||
match self.manager.activate(name).await {
|
match self.manager.activate(name).await {
|
||||||
Ok(result) => {
|
Ok(result) => {
|
||||||
@@ -471,10 +462,7 @@ impl Tool for ToolRemoveTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("name is required".to_string()))?;
|
|
||||||
|
|
||||||
let message = self
|
let message = self
|
||||||
.manager
|
.manager
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use async_trait::async_trait;
|
|||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::tools::tool::{Tool, ToolDomain, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolDomain, ToolError, ToolOutput, require_str};
|
||||||
use crate::workspace::paths as ws_paths;
|
use crate::workspace::paths as ws_paths;
|
||||||
|
|
||||||
/// Well-known workspace filenames that must go through memory_write, not write_file.
|
/// Well-known workspace filenames that must go through memory_write, not write_file.
|
||||||
@@ -203,10 +203,7 @@ impl Tool for ReadFileTool {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
_ctx: &JobContext,
|
_ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let path_str = params
|
let path_str = require_str(¶ms, "path")?;
|
||||||
.get("path")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'path' parameter".into()))?;
|
|
||||||
|
|
||||||
let offset = params.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
|
let offset = params.get("offset").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
|
||||||
let limit = params.get("limit").and_then(|v| v.as_u64());
|
let limit = params.get("limit").and_then(|v| v.as_u64());
|
||||||
@@ -328,10 +325,7 @@ impl Tool for WriteFileTool {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
_ctx: &JobContext,
|
_ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let path_str = params
|
let path_str = require_str(¶ms, "path")?;
|
||||||
.get("path")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'path' parameter".into()))?;
|
|
||||||
|
|
||||||
// Reject workspace paths: these live in the database, not on disk.
|
// Reject workspace paths: these live in the database, not on disk.
|
||||||
if is_workspace_path(path_str) {
|
if is_workspace_path(path_str) {
|
||||||
@@ -342,10 +336,7 @@ impl Tool for WriteFileTool {
|
|||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
let content = params
|
let content = require_str(¶ms, "content")?;
|
||||||
.get("content")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'content' parameter".into()))?;
|
|
||||||
|
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
@@ -650,20 +641,11 @@ impl Tool for ApplyPatchTool {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
_ctx: &JobContext,
|
_ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let path_str = params
|
let path_str = require_str(¶ms, "path")?;
|
||||||
.get("path")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'path' parameter".into()))?;
|
|
||||||
|
|
||||||
let old_string = params
|
let old_string = require_str(¶ms, "old_string")?;
|
||||||
.get("old_string")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'old_string' parameter".into()))?;
|
|
||||||
|
|
||||||
let new_string = params
|
let new_string = require_str(¶ms, "new_string")?;
|
||||||
.get("new_string")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'new_string' parameter".into()))?;
|
|
||||||
|
|
||||||
let replace_all = params
|
let replace_all = params
|
||||||
.get("replace_all")
|
.get("replace_all")
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use reqwest::Client;
|
|||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::safety::LeakDetector;
|
use crate::safety::LeakDetector;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
/// Maximum response body size (5 MB). Prevents OOM from unbounded responses.
|
/// Maximum response body size (5 MB). Prevents OOM from unbounded responses.
|
||||||
const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
|
const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
|
||||||
@@ -54,12 +54,12 @@ fn validate_url(url: &str) -> Result<reqwest::Url, ToolError> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check literal IP addresses
|
// Check literal IP addresses
|
||||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
if let Ok(ip) = host.parse::<IpAddr>()
|
||||||
if is_disallowed_ip(&ip) {
|
&& is_disallowed_ip(&ip)
|
||||||
return Err(ToolError::NotAuthorized(
|
{
|
||||||
"private or local IPs are not allowed".to_string(),
|
return Err(ToolError::NotAuthorized(
|
||||||
));
|
"private or local IPs are not allowed".to_string(),
|
||||||
}
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve hostname and check all resolved IPs against the blocklist.
|
// Resolve hostname and check all resolved IPs against the blocklist.
|
||||||
@@ -154,17 +154,9 @@ impl Tool for HttpTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let method = params
|
let method = require_str(¶ms, "method")?;
|
||||||
.get("method")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'method' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let url = params
|
let url = require_str(¶ms, "url")?;
|
||||||
.get("url")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'url' parameter".to_string()))?;
|
|
||||||
let parsed_url = validate_url(url)?;
|
let parsed_url = validate_url(url)?;
|
||||||
|
|
||||||
// Parse headers
|
// Parse headers
|
||||||
|
|||||||
+21
-34
@@ -15,9 +15,10 @@ use chrono::Utc;
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::context::{ContextManager, JobContext, JobState};
|
use crate::context::{ContextManager, JobContext, JobState};
|
||||||
use crate::history::{SandboxJobRecord, Store};
|
use crate::db::Database;
|
||||||
|
use crate::history::SandboxJobRecord;
|
||||||
use crate::orchestrator::job_manager::{ContainerJobManager, JobMode};
|
use crate::orchestrator::job_manager::{ContainerJobManager, JobMode};
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
/// Tool for creating a new job.
|
/// Tool for creating a new job.
|
||||||
///
|
///
|
||||||
@@ -27,7 +28,7 @@ use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
|||||||
pub struct CreateJobTool {
|
pub struct CreateJobTool {
|
||||||
context_manager: Arc<ContextManager>,
|
context_manager: Arc<ContextManager>,
|
||||||
job_manager: Option<Arc<ContainerJobManager>>,
|
job_manager: Option<Arc<ContainerJobManager>>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CreateJobTool {
|
impl CreateJobTool {
|
||||||
@@ -43,7 +44,7 @@ impl CreateJobTool {
|
|||||||
pub fn with_sandbox(
|
pub fn with_sandbox(
|
||||||
mut self,
|
mut self,
|
||||||
job_manager: Arc<ContainerJobManager>,
|
job_manager: Arc<ContainerJobManager>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
self.job_manager = Some(job_manager);
|
self.job_manager = Some(job_manager);
|
||||||
self.store = store;
|
self.store = store;
|
||||||
@@ -157,18 +158,18 @@ impl CreateJobTool {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Persist the job mode to DB
|
// Persist the job mode to DB
|
||||||
if mode == JobMode::ClaudeCode {
|
if mode == JobMode::ClaudeCode
|
||||||
if let Some(store) = self.store.clone() {
|
&& let Some(store) = self.store.clone()
|
||||||
let job_id_copy = job_id;
|
{
|
||||||
tokio::spawn(async move {
|
let job_id_copy = job_id;
|
||||||
if let Err(e) = store
|
tokio::spawn(async move {
|
||||||
.update_sandbox_job_mode(job_id_copy, "claude_code")
|
if let Err(e) = store
|
||||||
.await
|
.update_sandbox_job_mode(job_id_copy, "claude_code")
|
||||||
{
|
.await
|
||||||
tracing::warn!(job_id = %job_id_copy, "Failed to set job mode: {}", e);
|
{
|
||||||
}
|
tracing::warn!(job_id = %job_id_copy, "Failed to set job mode: {}", e);
|
||||||
});
|
}
|
||||||
}
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the container job with the pre-determined job_id.
|
// Create the container job with the pre-determined job_id.
|
||||||
@@ -466,17 +467,9 @@ impl Tool for CreateJobTool {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
ctx: &JobContext,
|
ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let title = params
|
let title = require_str(¶ms, "title")?;
|
||||||
.get("title")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'title' parameter".into()))?;
|
|
||||||
|
|
||||||
let description = params
|
let description = require_str(¶ms, "description")?;
|
||||||
.get("description")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'description' parameter".into())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if self.sandbox_enabled() {
|
if self.sandbox_enabled() {
|
||||||
let wait = params.get("wait").and_then(|v| v.as_bool()).unwrap_or(true);
|
let wait = params.get("wait").and_then(|v| v.as_bool()).unwrap_or(true);
|
||||||
@@ -634,10 +627,7 @@ impl Tool for JobStatusTool {
|
|||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
let requester_id = ctx.user_id.clone();
|
let requester_id = ctx.user_id.clone();
|
||||||
|
|
||||||
let job_id_str = params
|
let job_id_str = require_str(¶ms, "job_id")?;
|
||||||
.get("job_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'job_id' parameter".into()))?;
|
|
||||||
|
|
||||||
let job_id = Uuid::parse_str(job_id_str).map_err(|_| {
|
let job_id = Uuid::parse_str(job_id_str).map_err(|_| {
|
||||||
ToolError::InvalidParameters(format!("invalid job ID format: {}", job_id_str))
|
ToolError::InvalidParameters(format!("invalid job ID format: {}", job_id_str))
|
||||||
@@ -719,10 +709,7 @@ impl Tool for CancelJobTool {
|
|||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
let requester_id = ctx.user_id.clone();
|
let requester_id = ctx.user_id.clone();
|
||||||
|
|
||||||
let job_id_str = params
|
let job_id_str = require_str(¶ms, "job_id")?;
|
||||||
.get("job_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'job_id' parameter".into()))?;
|
|
||||||
|
|
||||||
let job_id = Uuid::parse_str(job_id_str).map_err(|_| {
|
let job_id = Uuid::parse_str(job_id_str).map_err(|_| {
|
||||||
ToolError::InvalidParameters(format!("invalid job ID format: {}", job_id_str))
|
ToolError::InvalidParameters(format!("invalid job ID format: {}", job_id_str))
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_param, require_str};
|
||||||
|
|
||||||
/// Tool for JSON manipulation (parse, query, transform).
|
/// Tool for JSON manipulation (parse, query, transform).
|
||||||
pub struct JsonTool;
|
pub struct JsonTool;
|
||||||
@@ -46,16 +46,9 @@ impl Tool for JsonTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let operation = params
|
let operation = require_str(¶ms, "operation")?;
|
||||||
.get("operation")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'operation' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let data = params
|
let data = require_param(¶ms, "data")?;
|
||||||
.get("data")
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'data' parameter".to_string()))?;
|
|
||||||
|
|
||||||
let result = match operation {
|
let result = match operation {
|
||||||
"parse" => {
|
"parse" => {
|
||||||
|
|||||||
@@ -1,160 +0,0 @@
|
|||||||
//! NEAR AI Marketplace tool.
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use rust_decimal::Decimal;
|
|
||||||
|
|
||||||
use crate::context::JobContext;
|
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
|
||||||
|
|
||||||
/// Tool for interacting with the NEAR AI marketplace.
|
|
||||||
pub struct MarketplaceTool {
|
|
||||||
// TODO: Add marketplace client
|
|
||||||
}
|
|
||||||
|
|
||||||
impl MarketplaceTool {
|
|
||||||
/// Create a new marketplace tool.
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for MarketplaceTool {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Tool for MarketplaceTool {
|
|
||||||
fn name(&self) -> &str {
|
|
||||||
"marketplace"
|
|
||||||
}
|
|
||||||
|
|
||||||
fn description(&self) -> &str {
|
|
||||||
"Interact with the NEAR AI marketplace: search jobs, submit bids, deliver work."
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parameters_schema(&self) -> serde_json::Value {
|
|
||||||
serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"action": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["search_jobs", "get_job", "submit_bid", "accept_job", "submit_work", "get_status"],
|
|
||||||
"description": "The marketplace action to perform"
|
|
||||||
},
|
|
||||||
"job_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Job ID (for get_job, submit_bid, accept_job, submit_work)"
|
|
||||||
},
|
|
||||||
"query": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Search query (for search_jobs)"
|
|
||||||
},
|
|
||||||
"category": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Job category filter (for search_jobs)"
|
|
||||||
},
|
|
||||||
"bid_amount": {
|
|
||||||
"type": "number",
|
|
||||||
"description": "Bid amount in NEAR (for submit_bid)"
|
|
||||||
},
|
|
||||||
"work_url": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "URL to submitted work (for submit_work)"
|
|
||||||
},
|
|
||||||
"work_description": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Description of completed work (for submit_work)"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["action"]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
params: serde_json::Value,
|
|
||||||
_ctx: &JobContext,
|
|
||||||
) -> Result<ToolOutput, ToolError> {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
|
|
||||||
let action = params
|
|
||||||
.get("action")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'action' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// TODO: Implement actual marketplace integration
|
|
||||||
let result = match action {
|
|
||||||
"search_jobs" => {
|
|
||||||
// Placeholder response
|
|
||||||
serde_json::json!({
|
|
||||||
"jobs": [],
|
|
||||||
"total": 0,
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_job" => {
|
|
||||||
let job_id = params
|
|
||||||
.get("job_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'job_id' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"job_id": job_id,
|
|
||||||
"status": "not_found",
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"submit_bid" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"success": false,
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"accept_job" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"success": false,
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"submit_work" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"success": false,
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_status" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"connected": false,
|
|
||||||
"message": "Marketplace integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(ToolError::InvalidParameters(format!(
|
|
||||||
"unknown action: {}",
|
|
||||||
action
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(ToolOutput::success(result, start.elapsed()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn estimated_cost(&self, params: &serde_json::Value) -> Option<Decimal> {
|
|
||||||
// Bidding has a cost
|
|
||||||
if params.get("action").and_then(|v| v.as_str()) == Some("submit_bid") {
|
|
||||||
Some(Decimal::new(1, 2)) // 0.01 NEAR gas cost
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn requires_sanitization(&self) -> bool {
|
|
||||||
true // External marketplace data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -17,7 +17,7 @@ use std::sync::Arc;
|
|||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
use crate::workspace::{Workspace, paths};
|
use crate::workspace::{Workspace, paths};
|
||||||
|
|
||||||
/// Identity files that the LLM must not overwrite via tool calls.
|
/// Identity files that the LLM must not overwrite via tool calls.
|
||||||
@@ -81,10 +81,7 @@ impl Tool for MemorySearchTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let query = params
|
let query = require_str(¶ms, "query")?;
|
||||||
.get("query")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'query' parameter".to_string()))?;
|
|
||||||
|
|
||||||
let limit = params
|
let limit = params
|
||||||
.get("limit")
|
.get("limit")
|
||||||
@@ -176,12 +173,7 @@ impl Tool for MemoryWriteTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let content = params
|
let content = require_str(¶ms, "content")?;
|
||||||
.get("content")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'content' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
if content.trim().is_empty() {
|
if content.trim().is_empty() {
|
||||||
return Err(ToolError::InvalidParameters(
|
return Err(ToolError::InvalidParameters(
|
||||||
@@ -337,10 +329,7 @@ impl Tool for MemoryReadTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let path = params
|
let path = require_str(¶ms, "path")?;
|
||||||
.get("path")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'path' parameter".to_string()))?;
|
|
||||||
|
|
||||||
let doc = self
|
let doc = self
|
||||||
.workspace
|
.workspace
|
||||||
@@ -482,7 +471,7 @@ impl Tool for MemoryTreeTool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(all(test, feature = "postgres"))]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
|||||||
@@ -1,22 +1,20 @@
|
|||||||
//! Built-in tools that come with the agent.
|
//! Built-in tools that come with the agent.
|
||||||
|
|
||||||
|
mod browser;
|
||||||
mod echo;
|
mod echo;
|
||||||
mod ecommerce;
|
|
||||||
pub mod extension_tools;
|
pub mod extension_tools;
|
||||||
mod file;
|
mod file;
|
||||||
mod http;
|
mod http;
|
||||||
mod job;
|
mod job;
|
||||||
mod json;
|
mod json;
|
||||||
mod marketplace;
|
|
||||||
mod memory;
|
mod memory;
|
||||||
mod restaurant;
|
|
||||||
pub mod routine;
|
pub mod routine;
|
||||||
pub(crate) mod shell;
|
pub(crate) mod shell;
|
||||||
mod taskrabbit;
|
|
||||||
mod time;
|
mod time;
|
||||||
|
|
||||||
|
pub use browser::BrowserTool;
|
||||||
|
pub use browser::session::find_chrome;
|
||||||
pub use echo::EchoTool;
|
pub use echo::EchoTool;
|
||||||
pub use ecommerce::EcommerceTool;
|
|
||||||
pub use extension_tools::{
|
pub use extension_tools::{
|
||||||
ToolActivateTool, ToolAuthTool, ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool,
|
ToolActivateTool, ToolAuthTool, ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool,
|
||||||
};
|
};
|
||||||
@@ -24,12 +22,9 @@ pub use file::{ApplyPatchTool, ListDirTool, ReadFileTool, WriteFileTool};
|
|||||||
pub use http::HttpTool;
|
pub use http::HttpTool;
|
||||||
pub use job::{CancelJobTool, CreateJobTool, JobStatusTool, ListJobsTool};
|
pub use job::{CancelJobTool, CreateJobTool, JobStatusTool, ListJobsTool};
|
||||||
pub use json::JsonTool;
|
pub use json::JsonTool;
|
||||||
pub use marketplace::MarketplaceTool;
|
|
||||||
pub use memory::{MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTool};
|
pub use memory::{MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTool};
|
||||||
pub use restaurant::RestaurantTool;
|
|
||||||
pub use routine::{
|
pub use routine::{
|
||||||
RoutineCreateTool, RoutineDeleteTool, RoutineHistoryTool, RoutineListTool, RoutineUpdateTool,
|
RoutineCreateTool, RoutineDeleteTool, RoutineHistoryTool, RoutineListTool, RoutineUpdateTool,
|
||||||
};
|
};
|
||||||
pub use shell::ShellTool;
|
pub use shell::ShellTool;
|
||||||
pub use taskrabbit::TaskRabbitTool;
|
|
||||||
pub use time::TimeTool;
|
pub use time::TimeTool;
|
||||||
|
|||||||
@@ -1,172 +0,0 @@
|
|||||||
//! Restaurant reservation tool.
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
|
|
||||||
use crate::context::JobContext;
|
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
|
||||||
|
|
||||||
/// Tool for restaurant reservations (OpenTable, Resy, etc.).
|
|
||||||
pub struct RestaurantTool {
|
|
||||||
// TODO: Add reservation API clients
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RestaurantTool {
|
|
||||||
/// Create a new restaurant tool.
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for RestaurantTool {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Tool for RestaurantTool {
|
|
||||||
fn name(&self) -> &str {
|
|
||||||
"restaurant"
|
|
||||||
}
|
|
||||||
|
|
||||||
fn description(&self) -> &str {
|
|
||||||
"Search restaurants, check availability, and make reservations via OpenTable, Resy, etc."
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parameters_schema(&self) -> serde_json::Value {
|
|
||||||
serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"action": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["search", "check_availability", "make_reservation", "cancel_reservation", "get_reservation"],
|
|
||||||
"description": "The restaurant action to perform"
|
|
||||||
},
|
|
||||||
"query": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Search query (cuisine type, restaurant name, etc.)"
|
|
||||||
},
|
|
||||||
"location": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"city": { "type": "string" },
|
|
||||||
"neighborhood": { "type": "string" },
|
|
||||||
"latitude": { "type": "number" },
|
|
||||||
"longitude": { "type": "number" }
|
|
||||||
},
|
|
||||||
"description": "Location to search near"
|
|
||||||
},
|
|
||||||
"date": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Reservation date (YYYY-MM-DD)"
|
|
||||||
},
|
|
||||||
"time": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Preferred time (HH:MM)"
|
|
||||||
},
|
|
||||||
"party_size": {
|
|
||||||
"type": "integer",
|
|
||||||
"description": "Number of guests"
|
|
||||||
},
|
|
||||||
"restaurant_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Restaurant ID (for check_availability, make_reservation)"
|
|
||||||
},
|
|
||||||
"reservation_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Reservation ID (for cancel_reservation, get_reservation)"
|
|
||||||
},
|
|
||||||
"guest_name": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Name for the reservation"
|
|
||||||
},
|
|
||||||
"guest_phone": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Phone number for the reservation"
|
|
||||||
},
|
|
||||||
"guest_email": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Email for the reservation"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["action"]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
params: serde_json::Value,
|
|
||||||
_ctx: &JobContext,
|
|
||||||
) -> Result<ToolOutput, ToolError> {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
|
|
||||||
let action = params
|
|
||||||
.get("action")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'action' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// TODO: Implement actual restaurant reservation API integrations
|
|
||||||
let result = match action {
|
|
||||||
"search" => {
|
|
||||||
let query = params.get("query").and_then(|v| v.as_str()).unwrap_or("");
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"query": query,
|
|
||||||
"restaurants": [],
|
|
||||||
"message": "Restaurant integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"check_availability" => {
|
|
||||||
let restaurant_id = params
|
|
||||||
.get("restaurant_id")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters(
|
|
||||||
"missing 'restaurant_id' parameter".to_string(),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"restaurant_id": restaurant_id,
|
|
||||||
"available_times": [],
|
|
||||||
"message": "Restaurant integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"make_reservation" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"success": false,
|
|
||||||
"message": "Restaurant integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"cancel_reservation" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"cancelled": false,
|
|
||||||
"message": "Restaurant integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_reservation" => {
|
|
||||||
let reservation_id = params.get("reservation_id").and_then(|v| v.as_str());
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"reservation_id": reservation_id,
|
|
||||||
"found": false,
|
|
||||||
"message": "Restaurant integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(ToolError::InvalidParameters(format!(
|
|
||||||
"unknown action: {}",
|
|
||||||
action
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(ToolOutput::success(result, start.elapsed()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn requires_sanitization(&self) -> bool {
|
|
||||||
true // External restaurant data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -19,18 +19,18 @@ use crate::agent::routine::{
|
|||||||
};
|
};
|
||||||
use crate::agent::routine_engine::RoutineEngine;
|
use crate::agent::routine_engine::RoutineEngine;
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::history::Store;
|
use crate::db::Database;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
// ==================== routine_create ====================
|
// ==================== routine_create ====================
|
||||||
|
|
||||||
pub struct RoutineCreateTool {
|
pub struct RoutineCreateTool {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
engine: Arc<RoutineEngine>,
|
engine: Arc<RoutineEngine>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutineCreateTool {
|
impl RoutineCreateTool {
|
||||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||||
Self { store, engine }
|
Self { store, engine }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -106,25 +106,16 @@ impl Tool for RoutineCreateTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'name'".to_string()))?;
|
|
||||||
|
|
||||||
let description = params
|
let description = params
|
||||||
.get("description")
|
.get("description")
|
||||||
.and_then(|v| v.as_str())
|
.and_then(|v| v.as_str())
|
||||||
.unwrap_or("");
|
.unwrap_or("");
|
||||||
|
|
||||||
let trigger_type = params
|
let trigger_type = require_str(¶ms, "trigger_type")?;
|
||||||
.get("trigger_type")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'trigger_type'".to_string()))?;
|
|
||||||
|
|
||||||
let prompt = params
|
let prompt = require_str(¶ms, "prompt")?;
|
||||||
.get("prompt")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'prompt'".to_string()))?;
|
|
||||||
|
|
||||||
// Build trigger
|
// Build trigger
|
||||||
let trigger = match trigger_type {
|
let trigger = match trigger_type {
|
||||||
@@ -277,11 +268,11 @@ impl Tool for RoutineCreateTool {
|
|||||||
// ==================== routine_list ====================
|
// ==================== routine_list ====================
|
||||||
|
|
||||||
pub struct RoutineListTool {
|
pub struct RoutineListTool {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutineListTool {
|
impl RoutineListTool {
|
||||||
pub fn new(store: Arc<Store>) -> Self {
|
pub fn new(store: Arc<dyn Database>) -> Self {
|
||||||
Self { store }
|
Self { store }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -351,12 +342,12 @@ impl Tool for RoutineListTool {
|
|||||||
// ==================== routine_update ====================
|
// ==================== routine_update ====================
|
||||||
|
|
||||||
pub struct RoutineUpdateTool {
|
pub struct RoutineUpdateTool {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
engine: Arc<RoutineEngine>,
|
engine: Arc<RoutineEngine>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutineUpdateTool {
|
impl RoutineUpdateTool {
|
||||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||||
Self { store, engine }
|
Self { store, engine }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -408,10 +399,7 @@ impl Tool for RoutineUpdateTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'name'".to_string()))?;
|
|
||||||
|
|
||||||
let mut routine = self
|
let mut routine = self
|
||||||
.store
|
.store
|
||||||
@@ -474,12 +462,12 @@ impl Tool for RoutineUpdateTool {
|
|||||||
// ==================== routine_delete ====================
|
// ==================== routine_delete ====================
|
||||||
|
|
||||||
pub struct RoutineDeleteTool {
|
pub struct RoutineDeleteTool {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
engine: Arc<RoutineEngine>,
|
engine: Arc<RoutineEngine>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutineDeleteTool {
|
impl RoutineDeleteTool {
|
||||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||||
Self { store, engine }
|
Self { store, engine }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -514,10 +502,7 @@ impl Tool for RoutineDeleteTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'name'".to_string()))?;
|
|
||||||
|
|
||||||
let routine = self
|
let routine = self
|
||||||
.store
|
.store
|
||||||
@@ -551,11 +536,11 @@ impl Tool for RoutineDeleteTool {
|
|||||||
// ==================== routine_history ====================
|
// ==================== routine_history ====================
|
||||||
|
|
||||||
pub struct RoutineHistoryTool {
|
pub struct RoutineHistoryTool {
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RoutineHistoryTool {
|
impl RoutineHistoryTool {
|
||||||
pub fn new(store: Arc<Store>) -> Self {
|
pub fn new(store: Arc<dyn Database>) -> Self {
|
||||||
Self { store }
|
Self { store }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -595,10 +580,7 @@ impl Tool for RoutineHistoryTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let name = params
|
let name = require_str(¶ms, "name")?;
|
||||||
.get("name")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'name'".to_string()))?;
|
|
||||||
|
|
||||||
let limit = params
|
let limit = params
|
||||||
.get("limit")
|
.get("limit")
|
||||||
|
|||||||
+50
-11
@@ -30,7 +30,7 @@ use tokio::process::Command;
|
|||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::sandbox::{SandboxManager, SandboxPolicy};
|
use crate::sandbox::{SandboxManager, SandboxPolicy};
|
||||||
use crate::tools::tool::{Tool, ToolDomain, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolDomain, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
/// Maximum output size before truncation (64KB).
|
/// Maximum output size before truncation (64KB).
|
||||||
const MAX_OUTPUT_SIZE: usize = 64 * 1024;
|
const MAX_OUTPUT_SIZE: usize = 64 * 1024;
|
||||||
@@ -343,12 +343,12 @@ impl ShellTool {
|
|||||||
|
|
||||||
// Use sandbox if configured; fail-closed (never silently fall through
|
// Use sandbox if configured; fail-closed (never silently fall through
|
||||||
// to unsandboxed execution when sandbox was intended).
|
// to unsandboxed execution when sandbox was intended).
|
||||||
if let Some(ref sandbox) = self.sandbox {
|
if let Some(ref sandbox) = self.sandbox
|
||||||
if sandbox.is_initialized() || sandbox.config().enabled {
|
&& (sandbox.is_initialized() || sandbox.config().enabled)
|
||||||
return self
|
{
|
||||||
.execute_sandboxed(sandbox, cmd, &cwd, timeout_duration)
|
return self
|
||||||
.await;
|
.execute_sandboxed(sandbox, cmd, &cwd, timeout_duration)
|
||||||
}
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only execute directly when no sandbox was configured at all.
|
// Only execute directly when no sandbox was configured at all.
|
||||||
@@ -401,10 +401,7 @@ impl Tool for ShellTool {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
_ctx: &JobContext,
|
_ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let command = params
|
let command = require_str(¶ms, "command")?;
|
||||||
.get("command")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| ToolError::InvalidParameters("missing 'command' parameter".into()))?;
|
|
||||||
|
|
||||||
let workdir = params.get("workdir").and_then(|v| v.as_str());
|
let workdir = params.get("workdir").and_then(|v| v.as_str());
|
||||||
let timeout = params.get("timeout").and_then(|v| v.as_u64());
|
let timeout = params.get("timeout").and_then(|v| v.as_u64());
|
||||||
@@ -527,6 +524,48 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Replicate the extraction logic from agent_loop.rs to prove it works
|
||||||
|
/// when `arguments` is a `serde_json::Value::Object` (the common case
|
||||||
|
/// that was previously broken because `Value::Object.as_str()` returns None).
|
||||||
|
#[test]
|
||||||
|
fn test_destructive_command_extraction_from_object_args() {
|
||||||
|
let arguments = serde_json::json!({"command": "rm -rf /tmp/stuff"});
|
||||||
|
|
||||||
|
let cmd = arguments
|
||||||
|
.get("command")
|
||||||
|
.and_then(|c| c.as_str().map(String::from))
|
||||||
|
.or_else(|| {
|
||||||
|
arguments
|
||||||
|
.as_str()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
|
||||||
|
.and_then(|v| v.get("command").and_then(|c| c.as_str().map(String::from)))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(cmd.as_deref(), Some("rm -rf /tmp/stuff"));
|
||||||
|
assert!(requires_explicit_approval(cmd.as_deref().unwrap()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Verify extraction still works when `arguments` is a JSON string
|
||||||
|
/// (rare, but possible if the LLM provider returns string-encoded JSON).
|
||||||
|
#[test]
|
||||||
|
fn test_destructive_command_extraction_from_string_args() {
|
||||||
|
let arguments =
|
||||||
|
serde_json::Value::String(r#"{"command": "git push --force origin main"}"#.to_string());
|
||||||
|
|
||||||
|
let cmd = arguments
|
||||||
|
.get("command")
|
||||||
|
.and_then(|c| c.as_str().map(String::from))
|
||||||
|
.or_else(|| {
|
||||||
|
arguments
|
||||||
|
.as_str()
|
||||||
|
.and_then(|s| serde_json::from_str::<serde_json::Value>(s).ok())
|
||||||
|
.and_then(|v| v.get("command").and_then(|c| c.as_str().map(String::from)))
|
||||||
|
});
|
||||||
|
|
||||||
|
assert_eq!(cmd.as_deref(), Some("git push --force origin main"));
|
||||||
|
assert!(requires_explicit_approval(cmd.as_deref().unwrap()));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_sandbox_policy_builder() {
|
fn test_sandbox_policy_builder() {
|
||||||
let tool = ShellTool::new()
|
let tool = ShellTool::new()
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
//! TaskRabbit tool for real-world task delegation.
|
|
||||||
|
|
||||||
use async_trait::async_trait;
|
|
||||||
use rust_decimal::Decimal;
|
|
||||||
|
|
||||||
use crate::context::JobContext;
|
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
|
||||||
|
|
||||||
/// Tool for delegating real-world tasks via TaskRabbit.
|
|
||||||
pub struct TaskRabbitTool {
|
|
||||||
// TODO: Add TaskRabbit API client
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TaskRabbitTool {
|
|
||||||
/// Create a new TaskRabbit tool.
|
|
||||||
pub fn new() -> Self {
|
|
||||||
Self {}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Default for TaskRabbitTool {
|
|
||||||
fn default() -> Self {
|
|
||||||
Self::new()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Tool for TaskRabbitTool {
|
|
||||||
fn name(&self) -> &str {
|
|
||||||
"taskrabbit"
|
|
||||||
}
|
|
||||||
|
|
||||||
fn description(&self) -> &str {
|
|
||||||
"Delegate real-world tasks to TaskRabbit taskers (delivery, assembly, cleaning, etc.)."
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parameters_schema(&self) -> serde_json::Value {
|
|
||||||
serde_json::json!({
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"action": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["search_taskers", "get_quote", "book_task", "get_status", "cancel_task"],
|
|
||||||
"description": "The TaskRabbit action to perform"
|
|
||||||
},
|
|
||||||
"task_type": {
|
|
||||||
"type": "string",
|
|
||||||
"enum": ["delivery", "assembly", "moving", "cleaning", "handyman", "other"],
|
|
||||||
"description": "Type of task"
|
|
||||||
},
|
|
||||||
"description": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Detailed description of the task"
|
|
||||||
},
|
|
||||||
"location": {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"address": { "type": "string" },
|
|
||||||
"city": { "type": "string" },
|
|
||||||
"state": { "type": "string" },
|
|
||||||
"zip": { "type": "string" }
|
|
||||||
},
|
|
||||||
"description": "Location for the task"
|
|
||||||
},
|
|
||||||
"scheduled_time": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "ISO 8601 datetime for when the task should be performed"
|
|
||||||
},
|
|
||||||
"budget": {
|
|
||||||
"type": "number",
|
|
||||||
"description": "Maximum budget for the task in USD"
|
|
||||||
},
|
|
||||||
"task_id": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "Task ID (for get_status, cancel_task)"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"required": ["action"]
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn execute(
|
|
||||||
&self,
|
|
||||||
params: serde_json::Value,
|
|
||||||
_ctx: &JobContext,
|
|
||||||
) -> Result<ToolOutput, ToolError> {
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
|
|
||||||
let action = params
|
|
||||||
.get("action")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'action' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
// TODO: Implement actual TaskRabbit API integration
|
|
||||||
let result = match action {
|
|
||||||
"search_taskers" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"taskers": [],
|
|
||||||
"message": "TaskRabbit integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_quote" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"quotes": [],
|
|
||||||
"message": "TaskRabbit integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"book_task" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"booked": false,
|
|
||||||
"message": "TaskRabbit integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"get_status" => {
|
|
||||||
let task_id = params.get("task_id").and_then(|v| v.as_str());
|
|
||||||
|
|
||||||
serde_json::json!({
|
|
||||||
"task_id": task_id,
|
|
||||||
"status": "unknown",
|
|
||||||
"message": "TaskRabbit integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
"cancel_task" => {
|
|
||||||
serde_json::json!({
|
|
||||||
"cancelled": false,
|
|
||||||
"message": "TaskRabbit integration not yet implemented"
|
|
||||||
})
|
|
||||||
}
|
|
||||||
_ => {
|
|
||||||
return Err(ToolError::InvalidParameters(format!(
|
|
||||||
"unknown action: {}",
|
|
||||||
action
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Ok(ToolOutput::success(result, start.elapsed()))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn estimated_cost(&self, params: &serde_json::Value) -> Option<Decimal> {
|
|
||||||
// Booking a task has associated costs
|
|
||||||
if params.get("action").and_then(|v| v.as_str()) == Some("book_task") {
|
|
||||||
params
|
|
||||||
.get("budget")
|
|
||||||
.and_then(|v| v.as_f64())
|
|
||||||
.map(|b| Decimal::try_from(b).unwrap_or_default())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn requires_sanitization(&self) -> bool {
|
|
||||||
true // External TaskRabbit data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -4,7 +4,7 @@ use async_trait::async_trait;
|
|||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
|
||||||
use crate::context::JobContext;
|
use crate::context::JobContext;
|
||||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||||
|
|
||||||
/// Tool for getting current time and date operations.
|
/// Tool for getting current time and date operations.
|
||||||
pub struct TimeTool;
|
pub struct TimeTool;
|
||||||
@@ -52,12 +52,7 @@ impl Tool for TimeTool {
|
|||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let start = std::time::Instant::now();
|
let start = std::time::Instant::now();
|
||||||
|
|
||||||
let operation = params
|
let operation = require_str(¶ms, "operation")?;
|
||||||
.get("operation")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'operation' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let result = match operation {
|
let result = match operation {
|
||||||
"now" => {
|
"now" => {
|
||||||
@@ -69,12 +64,7 @@ impl Tool for TimeTool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
"parse" => {
|
"parse" => {
|
||||||
let timestamp = params
|
let timestamp = require_str(¶ms, "timestamp")?;
|
||||||
.get("timestamp")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'timestamp' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let dt: DateTime<Utc> = timestamp.parse().map_err(|e| {
|
let dt: DateTime<Utc> = timestamp.parse().map_err(|e| {
|
||||||
ToolError::InvalidParameters(format!("invalid timestamp: {}", e))
|
ToolError::InvalidParameters(format!("invalid timestamp: {}", e))
|
||||||
@@ -87,19 +77,9 @@ impl Tool for TimeTool {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
"diff" => {
|
"diff" => {
|
||||||
let ts1 = params
|
let ts1 = require_str(¶ms, "timestamp")?;
|
||||||
.get("timestamp")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'timestamp' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let ts2 = params
|
let ts2 = require_str(¶ms, "timestamp2")?;
|
||||||
.get("timestamp2")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'timestamp2' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let dt1: DateTime<Utc> = ts1.parse().map_err(|e| {
|
let dt1: DateTime<Utc> = ts1.parse().map_err(|e| {
|
||||||
ToolError::InvalidParameters(format!("invalid timestamp: {}", e))
|
ToolError::InvalidParameters(format!("invalid timestamp: {}", e))
|
||||||
|
|||||||
+15
-72
@@ -11,9 +11,9 @@ use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
|||||||
use rand::RngCore;
|
use rand::RngCore;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
|
|
||||||
|
use crate::cli::oauth_defaults::{self, OAUTH_CALLBACK_PORT};
|
||||||
use crate::secrets::{CreateSecretParams, SecretsStore};
|
use crate::secrets::{CreateSecretParams, SecretsStore};
|
||||||
use crate::tools::mcp::config::McpServerConfig;
|
use crate::tools::mcp::config::McpServerConfig;
|
||||||
|
|
||||||
@@ -466,14 +466,12 @@ pub async fn authorize_mcp_server(
|
|||||||
Ok(token)
|
Ok(token)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find an available port for the OAuth callback.
|
/// Bind the OAuth callback listener on the shared fixed port.
|
||||||
pub async fn find_available_port() -> Result<(TcpListener, u16), AuthError> {
|
pub async fn find_available_port() -> Result<(TcpListener, u16), AuthError> {
|
||||||
for port in 9876..=9886 {
|
let listener = oauth_defaults::bind_callback_listener()
|
||||||
if let Ok(listener) = TcpListener::bind(format!("127.0.0.1:{}", port)).await {
|
.await
|
||||||
return Ok((listener, port));
|
.map_err(|_| AuthError::PortUnavailable)?;
|
||||||
}
|
Ok((listener, OAUTH_CALLBACK_PORT))
|
||||||
}
|
|
||||||
Err(AuthError::PortUnavailable)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the authorization URL with all required parameters.
|
/// Build the authorization URL with all required parameters.
|
||||||
@@ -522,71 +520,16 @@ pub async fn wait_for_authorization_callback(
|
|||||||
listener: TcpListener,
|
listener: TcpListener,
|
||||||
server_name: &str,
|
server_name: &str,
|
||||||
) -> Result<String, AuthError> {
|
) -> Result<String, AuthError> {
|
||||||
let timeout = Duration::from_secs(300);
|
oauth_defaults::wait_for_callback(listener, "/callback", "code", server_name)
|
||||||
|
.await
|
||||||
tokio::time::timeout(timeout, async {
|
.map_err(|e| match e {
|
||||||
loop {
|
oauth_defaults::OAuthCallbackError::Denied => AuthError::AuthorizationDenied,
|
||||||
let (mut socket, _) = listener
|
oauth_defaults::OAuthCallbackError::Timeout => AuthError::Timeout,
|
||||||
.accept()
|
oauth_defaults::OAuthCallbackError::PortInUse(_, msg) => {
|
||||||
.await
|
AuthError::Http(format!("Port error: {}", msg))
|
||||||
.map_err(|e| AuthError::Http(e.to_string()))?;
|
|
||||||
|
|
||||||
let mut reader = BufReader::new(&mut socket);
|
|
||||||
let mut request_line = String::new();
|
|
||||||
reader
|
|
||||||
.read_line(&mut request_line)
|
|
||||||
.await
|
|
||||||
.map_err(|e| AuthError::Http(e.to_string()))?;
|
|
||||||
|
|
||||||
// Parse GET /callback?code=xxx HTTP/1.1
|
|
||||||
if let Some(path) = request_line.split_whitespace().nth(1) {
|
|
||||||
if path.starts_with("/callback") {
|
|
||||||
if let Some(query) = path.split('?').nth(1) {
|
|
||||||
// Check for error first
|
|
||||||
if query.contains("error=") {
|
|
||||||
let response = "HTTP/1.1 400 Bad Request\r\n\r\nAuthorization denied";
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
return Err(AuthError::AuthorizationDenied);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Look for code
|
|
||||||
for param in query.split('&') {
|
|
||||||
let parts: Vec<&str> = param.splitn(2, '=').collect();
|
|
||||||
if parts.len() == 2 && parts[0] == "code" {
|
|
||||||
let code = urlencoding::decode(parts[1])
|
|
||||||
.unwrap_or_else(|_| parts[1].into())
|
|
||||||
.into_owned();
|
|
||||||
|
|
||||||
// Send success response
|
|
||||||
let response = format!(
|
|
||||||
"HTTP/1.1 200 OK\r\n\
|
|
||||||
Content-Type: text/html\r\n\
|
|
||||||
\r\n\
|
|
||||||
<!DOCTYPE html><html><body style=\"font-family: sans-serif; \
|
|
||||||
display: flex; justify-content: center; align-items: center; \
|
|
||||||
height: 100vh; margin: 0; background: #191919; color: white;\">\
|
|
||||||
<div style=\"text-align: center;\">\
|
|
||||||
<h1>✓ {} Connected!</h1>\
|
|
||||||
<p>You can close this window.</p>\
|
|
||||||
</div></body></html>",
|
|
||||||
server_name
|
|
||||||
);
|
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
let _ = socket.shutdown().await;
|
|
||||||
|
|
||||||
return Ok(code);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
oauth_defaults::OAuthCallbackError::Io(msg) => AuthError::Http(msg),
|
||||||
let response = "HTTP/1.1 404 Not Found\r\n\r\n";
|
})
|
||||||
let _ = socket.write_all(response.as_bytes()).await;
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|_| AuthError::Timeout)?
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exchange the authorization code for an access token.
|
/// Exchange the authorization code for an access token.
|
||||||
|
|||||||
+44
-43
@@ -184,44 +184,46 @@ impl McpClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add Mcp-Session-Id header if we have a session
|
// Add Mcp-Session-Id header if we have a session
|
||||||
if let Some(ref session_manager) = self.session_manager {
|
if let Some(ref session_manager) = self.session_manager
|
||||||
if let Some(session_id) = session_manager.get_session_id(&self.server_name).await {
|
&& let Some(session_id) = session_manager.get_session_id(&self.server_name).await
|
||||||
req_builder = req_builder.header("Mcp-Session-Id", session_id);
|
{
|
||||||
}
|
req_builder = req_builder.header("Mcp-Session-Id", session_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
let response = req_builder
|
let response = req_builder.send().await.map_err(|e| {
|
||||||
.send()
|
let mut chain = format!("MCP request failed: {}", e);
|
||||||
.await
|
let mut source = std::error::Error::source(&e);
|
||||||
.map_err(|e| ToolError::ExternalService(format!("MCP request failed: {}", e)))?;
|
while let Some(cause) = source {
|
||||||
|
chain.push_str(&format!(" -> {}", cause));
|
||||||
|
source = cause.source();
|
||||||
|
}
|
||||||
|
ToolError::ExternalService(chain)
|
||||||
|
})?;
|
||||||
|
|
||||||
// Check for 401 Unauthorized - try to refresh token on first attempt
|
// Check for 401 Unauthorized - try to refresh token on first attempt
|
||||||
if response.status() == reqwest::StatusCode::UNAUTHORIZED {
|
if response.status() == reqwest::StatusCode::UNAUTHORIZED {
|
||||||
if attempt == 0 {
|
if attempt == 0 {
|
||||||
// Try to refresh the token
|
// Try to refresh the token
|
||||||
if let Some(ref secrets) = self.secrets {
|
if let Some(ref secrets) = self.secrets
|
||||||
if let Some(ref config) = self.server_config {
|
&& let Some(ref config) = self.server_config
|
||||||
tracing::debug!(
|
{
|
||||||
"MCP token expired, attempting refresh for '{}'",
|
tracing::debug!(
|
||||||
self.server_name
|
"MCP token expired, attempting refresh for '{}'",
|
||||||
);
|
self.server_name
|
||||||
match refresh_access_token(config, secrets, &self.user_id).await {
|
);
|
||||||
Ok(_) => {
|
match refresh_access_token(config, secrets, &self.user_id).await {
|
||||||
tracing::info!(
|
Ok(_) => {
|
||||||
"MCP token refreshed for '{}'",
|
tracing::info!("MCP token refreshed for '{}'", self.server_name);
|
||||||
self.server_name
|
// Continue to next iteration to retry with new token
|
||||||
);
|
continue;
|
||||||
// Continue to next iteration to retry with new token
|
}
|
||||||
continue;
|
Err(e) => {
|
||||||
}
|
tracing::debug!(
|
||||||
Err(e) => {
|
"Token refresh failed for '{}': {}",
|
||||||
tracing::debug!(
|
self.server_name,
|
||||||
"Token refresh failed for '{}': {}",
|
e
|
||||||
self.server_name,
|
);
|
||||||
e
|
// Fall through to return auth error
|
||||||
);
|
|
||||||
// Fall through to return auth error
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -245,16 +247,15 @@ impl McpClient {
|
|||||||
/// Parse the HTTP response into an MCP response.
|
/// Parse the HTTP response into an MCP response.
|
||||||
async fn parse_response(&self, response: reqwest::Response) -> Result<McpResponse, ToolError> {
|
async fn parse_response(&self, response: reqwest::Response) -> Result<McpResponse, ToolError> {
|
||||||
// Extract session ID from response header
|
// Extract session ID from response header
|
||||||
if let Some(ref session_manager) = self.session_manager {
|
if let Some(ref session_manager) = self.session_manager
|
||||||
if let Some(session_id) = response
|
&& let Some(session_id) = response
|
||||||
.headers()
|
.headers()
|
||||||
.get("Mcp-Session-Id")
|
.get("Mcp-Session-Id")
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
{
|
{
|
||||||
session_manager
|
session_manager
|
||||||
.update_session_id(&self.server_name, Some(session_id.to_string()))
|
.update_session_id(&self.server_name, Some(session_id.to_string()))
|
||||||
.await;
|
.await;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !response.status().is_success() {
|
if !response.status().is_success() {
|
||||||
@@ -316,11 +317,11 @@ impl McpClient {
|
|||||||
/// This should be called once per session to establish capabilities.
|
/// This should be called once per session to establish capabilities.
|
||||||
pub async fn initialize(&self) -> Result<InitializeResult, ToolError> {
|
pub async fn initialize(&self) -> Result<InitializeResult, ToolError> {
|
||||||
// Check if already initialized
|
// Check if already initialized
|
||||||
if let Some(ref session_manager) = self.session_manager {
|
if let Some(ref session_manager) = self.session_manager
|
||||||
if session_manager.is_initialized(&self.server_name).await {
|
&& session_manager.is_initialized(&self.server_name).await
|
||||||
// Return cached/default capabilities
|
{
|
||||||
return Ok(InitializeResult::default());
|
// Return cached/default capabilities
|
||||||
}
|
return Ok(InitializeResult::default());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure we have a session
|
// Ensure we have a session
|
||||||
|
|||||||
+86
-5
@@ -88,8 +88,18 @@ impl McpServerConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Check if this server requires authentication.
|
/// Check if this server requires authentication.
|
||||||
|
///
|
||||||
|
/// Returns true if OAuth is pre-configured OR if this is a remote HTTPS server
|
||||||
|
/// (which likely supports Dynamic Client Registration even without pre-configured OAuth).
|
||||||
pub fn requires_auth(&self) -> bool {
|
pub fn requires_auth(&self) -> bool {
|
||||||
self.oauth.is_some()
|
if self.oauth.is_some() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// Remote HTTPS servers need auth handling (DCR, token refresh, 401 detection).
|
||||||
|
// Localhost/127.0.0.1 servers are assumed to be dev servers without auth.
|
||||||
|
let url_lower = self.url.to_lowercase();
|
||||||
|
let is_localhost = is_localhost_url(&url_lower);
|
||||||
|
url_lower.starts_with("https://") && !is_localhost
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the secret name used to store the access token.
|
/// Get the secret name used to store the access token.
|
||||||
@@ -333,7 +343,7 @@ pub async fn get_mcp_server(name: &str) -> Result<McpServerConfig, ConfigError>
|
|||||||
///
|
///
|
||||||
/// Falls back to the disk file if DB has no entry.
|
/// Falls back to the disk file if DB has no entry.
|
||||||
pub async fn load_mcp_servers_from_db(
|
pub async fn load_mcp_servers_from_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
) -> Result<McpServersFile, ConfigError> {
|
) -> Result<McpServersFile, ConfigError> {
|
||||||
match store.get_setting(user_id, "mcp_servers").await {
|
match store.get_setting(user_id, "mcp_servers").await {
|
||||||
@@ -357,7 +367,7 @@ pub async fn load_mcp_servers_from_db(
|
|||||||
|
|
||||||
/// Save MCP server configurations to the database settings table.
|
/// Save MCP server configurations to the database settings table.
|
||||||
pub async fn save_mcp_servers_to_db(
|
pub async fn save_mcp_servers_to_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
config: &McpServersFile,
|
config: &McpServersFile,
|
||||||
) -> Result<(), ConfigError> {
|
) -> Result<(), ConfigError> {
|
||||||
@@ -371,7 +381,7 @@ pub async fn save_mcp_servers_to_db(
|
|||||||
|
|
||||||
/// Add a new MCP server configuration (DB-backed).
|
/// Add a new MCP server configuration (DB-backed).
|
||||||
pub async fn add_mcp_server_db(
|
pub async fn add_mcp_server_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
config: McpServerConfig,
|
config: McpServerConfig,
|
||||||
) -> Result<(), ConfigError> {
|
) -> Result<(), ConfigError> {
|
||||||
@@ -386,7 +396,7 @@ pub async fn add_mcp_server_db(
|
|||||||
|
|
||||||
/// Remove an MCP server by name (DB-backed).
|
/// Remove an MCP server by name (DB-backed).
|
||||||
pub async fn remove_mcp_server_db(
|
pub async fn remove_mcp_server_db(
|
||||||
store: &crate::history::Store,
|
store: &dyn crate::db::Database,
|
||||||
user_id: &str,
|
user_id: &str,
|
||||||
name: &str,
|
name: &str,
|
||||||
) -> Result<(), ConfigError> {
|
) -> Result<(), ConfigError> {
|
||||||
@@ -402,11 +412,43 @@ pub async fn remove_mcp_server_db(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Check if a URL points to a loopback address (localhost, 127.0.0.1, [::1]).
|
||||||
|
///
|
||||||
|
/// Uses `url::Url` for proper parsing so edge cases (IPv6, userinfo, ports)
|
||||||
|
/// are handled correctly without manual string splitting.
|
||||||
|
fn is_localhost_url(url: &str) -> bool {
|
||||||
|
let Ok(parsed) = url::Url::parse(url) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
match parsed.host() {
|
||||||
|
Some(url::Host::Domain(d)) => d.eq_ignore_ascii_case("localhost"),
|
||||||
|
Some(url::Host::Ipv4(ip)) => ip.is_loopback(),
|
||||||
|
Some(url::Host::Ipv6(ip)) => ip.is_loopback(),
|
||||||
|
None => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use tempfile::tempdir;
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_is_localhost_url() {
|
||||||
|
assert!(is_localhost_url("http://localhost:3000/path"));
|
||||||
|
assert!(is_localhost_url("https://localhost/path"));
|
||||||
|
assert!(is_localhost_url("http://127.0.0.1:8080"));
|
||||||
|
assert!(is_localhost_url("http://127.0.0.1"));
|
||||||
|
assert!(!is_localhost_url("https://notlocalhost.com/path"));
|
||||||
|
assert!(!is_localhost_url("https://example-localhost.io"));
|
||||||
|
assert!(!is_localhost_url("https://mcp.notion.com"));
|
||||||
|
assert!(is_localhost_url("http://user:pass@localhost:3000/path"));
|
||||||
|
// IPv6 loopback
|
||||||
|
assert!(is_localhost_url("http://[::1]:8080/path"));
|
||||||
|
assert!(is_localhost_url("http://[::1]/path"));
|
||||||
|
assert!(!is_localhost_url("http://[::2]:8080/path"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_server_config_validation() {
|
fn test_server_config_validation() {
|
||||||
// Valid HTTPS server
|
// Valid HTTPS server
|
||||||
@@ -514,4 +556,43 @@ mod tests {
|
|||||||
"mcp_notion_refresh_token"
|
"mcp_notion_refresh_token"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_requires_auth_with_oauth() {
|
||||||
|
let config = McpServerConfig::new("notion", "https://mcp.notion.com")
|
||||||
|
.with_oauth(OAuthConfig::new("client-123"));
|
||||||
|
assert!(config.requires_auth());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_requires_auth_remote_https_without_oauth() {
|
||||||
|
// Remote HTTPS servers need auth even without pre-configured OAuth (DCR)
|
||||||
|
let config = McpServerConfig::new("github-copilot", "https://api.githubcopilot.com/mcp/");
|
||||||
|
assert!(config.requires_auth());
|
||||||
|
|
||||||
|
let config = McpServerConfig::new("notion", "https://mcp.notion.com");
|
||||||
|
assert!(config.requires_auth());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_requires_auth_localhost_no_auth() {
|
||||||
|
// Localhost servers are dev servers, no auth needed
|
||||||
|
let config = McpServerConfig::new("local", "http://localhost:8080");
|
||||||
|
assert!(!config.requires_auth());
|
||||||
|
|
||||||
|
let config = McpServerConfig::new("local", "http://127.0.0.1:3000/mcp");
|
||||||
|
assert!(!config.requires_auth());
|
||||||
|
|
||||||
|
// Even HTTPS localhost doesn't require auth
|
||||||
|
let config = McpServerConfig::new("local", "https://localhost:8443");
|
||||||
|
assert!(!config.requires_auth());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_requires_auth_http_remote_no_auth() {
|
||||||
|
// HTTP remote servers won't pass validation, but if they existed
|
||||||
|
// they wouldn't trigger HTTPS auth detection
|
||||||
|
let config = McpServerConfig::new("bad", "http://mcp.example.com");
|
||||||
|
assert!(!config.requires_auth());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+28
-14
@@ -6,22 +6,23 @@ use std::sync::Arc;
|
|||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
|
||||||
use crate::context::ContextManager;
|
use crate::context::ContextManager;
|
||||||
|
use crate::db::Database;
|
||||||
use crate::extensions::ExtensionManager;
|
use crate::extensions::ExtensionManager;
|
||||||
use crate::history::Store;
|
|
||||||
use crate::llm::{LlmProvider, ToolDefinition};
|
use crate::llm::{LlmProvider, ToolDefinition};
|
||||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||||
use crate::safety::SafetyLayer;
|
use crate::safety::SafetyLayer;
|
||||||
|
use crate::secrets::SecretsStore;
|
||||||
use crate::tools::builder::{BuildSoftwareTool, BuilderConfig, LlmSoftwareBuilder};
|
use crate::tools::builder::{BuildSoftwareTool, BuilderConfig, LlmSoftwareBuilder};
|
||||||
use crate::tools::builtin::{
|
use crate::tools::builtin::{
|
||||||
ApplyPatchTool, CancelJobTool, CreateJobTool, EchoTool, HttpTool, JobStatusTool, JsonTool,
|
ApplyPatchTool, BrowserTool, CancelJobTool, CreateJobTool, EchoTool, HttpTool, JobStatusTool,
|
||||||
ListDirTool, ListJobsTool, MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTool,
|
JsonTool, ListDirTool, ListJobsTool, MemoryReadTool, MemorySearchTool, MemoryTreeTool,
|
||||||
ReadFileTool, ShellTool, TimeTool, ToolActivateTool, ToolAuthTool, ToolInstallTool,
|
MemoryWriteTool, ReadFileTool, ShellTool, TimeTool, ToolActivateTool, ToolAuthTool,
|
||||||
ToolListTool, ToolRemoveTool, ToolSearchTool, WriteFileTool,
|
ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool, WriteFileTool,
|
||||||
};
|
};
|
||||||
use crate::tools::tool::{Tool, ToolDomain};
|
use crate::tools::tool::{Tool, ToolDomain};
|
||||||
use crate::tools::wasm::{
|
use crate::tools::wasm::{
|
||||||
Capabilities, ResourceLimits, WasmError, WasmStorageError, WasmToolRuntime, WasmToolStore,
|
Capabilities, OAuthRefreshConfig, ResourceLimits, WasmError, WasmStorageError, WasmToolRuntime,
|
||||||
WasmToolWrapper,
|
WasmToolStore, WasmToolWrapper,
|
||||||
};
|
};
|
||||||
use crate::workspace::Workspace;
|
use crate::workspace::Workspace;
|
||||||
|
|
||||||
@@ -96,10 +97,10 @@ impl ToolRegistry {
|
|||||||
if let Ok(mut tools) = self.tools.try_write() {
|
if let Ok(mut tools) = self.tools.try_write() {
|
||||||
tools.insert(name.clone(), tool);
|
tools.insert(name.clone(), tool);
|
||||||
// Mark as built-in so it can't be shadowed later
|
// Mark as built-in so it can't be shadowed later
|
||||||
if PROTECTED_TOOL_NAMES.contains(&name.as_str()) {
|
if PROTECTED_TOOL_NAMES.contains(&name.as_str())
|
||||||
if let Ok(mut builtins) = self.builtin_names.try_write() {
|
&& let Ok(mut builtins) = self.builtin_names.try_write()
|
||||||
builtins.insert(name.clone());
|
{
|
||||||
}
|
builtins.insert(name.clone());
|
||||||
}
|
}
|
||||||
tracing::debug!("Registered tool: {}", name);
|
tracing::debug!("Registered tool: {}", name);
|
||||||
}
|
}
|
||||||
@@ -217,8 +218,9 @@ impl ToolRegistry {
|
|||||||
self.register_sync(Arc::new(WriteFileTool::new()));
|
self.register_sync(Arc::new(WriteFileTool::new()));
|
||||||
self.register_sync(Arc::new(ListDirTool::new()));
|
self.register_sync(Arc::new(ListDirTool::new()));
|
||||||
self.register_sync(Arc::new(ApplyPatchTool::new()));
|
self.register_sync(Arc::new(ApplyPatchTool::new()));
|
||||||
|
self.register_sync(Arc::new(BrowserTool::new()));
|
||||||
|
|
||||||
tracing::info!("Registered 5 development tools");
|
tracing::info!("Registered 6 development tools (includes browser)");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Register memory tools with a workspace.
|
/// Register memory tools with a workspace.
|
||||||
@@ -243,7 +245,7 @@ impl ToolRegistry {
|
|||||||
&self,
|
&self,
|
||||||
context_manager: Arc<ContextManager>,
|
context_manager: Arc<ContextManager>,
|
||||||
job_manager: Option<Arc<ContainerJobManager>>,
|
job_manager: Option<Arc<ContainerJobManager>>,
|
||||||
store: Option<Arc<Store>>,
|
store: Option<Arc<dyn Database>>,
|
||||||
) {
|
) {
|
||||||
let mut create_tool = CreateJobTool::new(Arc::clone(&context_manager));
|
let mut create_tool = CreateJobTool::new(Arc::clone(&context_manager));
|
||||||
if let Some(jm) = job_manager {
|
if let Some(jm) = job_manager {
|
||||||
@@ -276,7 +278,7 @@ impl ToolRegistry {
|
|||||||
/// of routines (scheduled and event-driven tasks).
|
/// of routines (scheduled and event-driven tasks).
|
||||||
pub fn register_routine_tools(
|
pub fn register_routine_tools(
|
||||||
&self,
|
&self,
|
||||||
store: Arc<Store>,
|
store: Arc<dyn Database>,
|
||||||
engine: Arc<crate::agent::routine_engine::RoutineEngine>,
|
engine: Arc<crate::agent::routine_engine::RoutineEngine>,
|
||||||
) {
|
) {
|
||||||
use crate::tools::builtin::{
|
use crate::tools::builtin::{
|
||||||
@@ -366,6 +368,12 @@ impl ToolRegistry {
|
|||||||
if let Some(s) = reg.schema {
|
if let Some(s) = reg.schema {
|
||||||
wrapper = wrapper.with_schema(s);
|
wrapper = wrapper.with_schema(s);
|
||||||
}
|
}
|
||||||
|
if let Some(store) = reg.secrets_store {
|
||||||
|
wrapper = wrapper.with_secrets_store(store);
|
||||||
|
}
|
||||||
|
if let Some(oauth) = reg.oauth_refresh {
|
||||||
|
wrapper = wrapper.with_oauth_refresh(oauth);
|
||||||
|
}
|
||||||
|
|
||||||
// Register the tool
|
// Register the tool
|
||||||
self.register(Arc::new(wrapper)).await;
|
self.register(Arc::new(wrapper)).await;
|
||||||
@@ -421,6 +429,8 @@ impl ToolRegistry {
|
|||||||
limits: None,
|
limits: None,
|
||||||
description: Some(&tool_with_binary.tool.description),
|
description: Some(&tool_with_binary.tool.description),
|
||||||
schema: Some(tool_with_binary.tool.parameters_schema.clone()),
|
schema: Some(tool_with_binary.tool.parameters_schema.clone()),
|
||||||
|
secrets_store: None,
|
||||||
|
oauth_refresh: None,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(WasmRegistrationError::Wasm)?;
|
.map_err(WasmRegistrationError::Wasm)?;
|
||||||
@@ -462,6 +472,10 @@ pub struct WasmToolRegistration<'a> {
|
|||||||
pub description: Option<&'a str>,
|
pub description: Option<&'a str>,
|
||||||
/// Optional parameter schema override.
|
/// Optional parameter schema override.
|
||||||
pub schema: Option<serde_json::Value>,
|
pub schema: Option<serde_json::Value>,
|
||||||
|
/// Secrets store for credential injection at request time.
|
||||||
|
pub secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>,
|
||||||
|
/// OAuth refresh configuration for auto-refreshing expired tokens.
|
||||||
|
pub oauth_refresh: Option<OAuthRefreshConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for ToolRegistry {
|
impl Default for ToolRegistry {
|
||||||
|
|||||||
+59
-6
@@ -199,6 +199,28 @@ pub trait Tool: Send + Sync {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extract a required string parameter from a JSON object.
|
||||||
|
///
|
||||||
|
/// Returns `ToolError::InvalidParameters` if the key is missing or not a string.
|
||||||
|
pub fn require_str<'a>(params: &'a serde_json::Value, name: &str) -> Result<&'a str, ToolError> {
|
||||||
|
params
|
||||||
|
.get(name)
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.ok_or_else(|| ToolError::InvalidParameters(format!("missing '{}' parameter", name)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract a required parameter of any type from a JSON object.
|
||||||
|
///
|
||||||
|
/// Returns `ToolError::InvalidParameters` if the key is missing.
|
||||||
|
pub fn require_param<'a>(
|
||||||
|
params: &'a serde_json::Value,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<&'a serde_json::Value, ToolError> {
|
||||||
|
params
|
||||||
|
.get(name)
|
||||||
|
.ok_or_else(|| ToolError::InvalidParameters(format!("missing '{}' parameter", name)))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
@@ -235,12 +257,7 @@ mod tests {
|
|||||||
params: serde_json::Value,
|
params: serde_json::Value,
|
||||||
_ctx: &JobContext,
|
_ctx: &JobContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let message = params
|
let message = require_str(¶ms, "message")?;
|
||||||
.get("message")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ToolError::InvalidParameters("missing 'message' parameter".to_string())
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(ToolOutput::text(message, Duration::from_millis(1)))
|
Ok(ToolOutput::text(message, Duration::from_millis(1)))
|
||||||
}
|
}
|
||||||
@@ -277,4 +294,40 @@ mod tests {
|
|||||||
let tool = EchoTool;
|
let tool = EchoTool;
|
||||||
assert_eq!(tool.execution_timeout(), Duration::from_secs(60));
|
assert_eq!(tool.execution_timeout(), Duration::from_secs(60));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_require_str_present() {
|
||||||
|
let params = serde_json::json!({"name": "alice"});
|
||||||
|
assert_eq!(require_str(¶ms, "name").unwrap(), "alice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_require_str_missing() {
|
||||||
|
let params = serde_json::json!({});
|
||||||
|
let err = require_str(¶ms, "name").unwrap_err();
|
||||||
|
assert!(err.to_string().contains("missing 'name'"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_require_str_wrong_type() {
|
||||||
|
let params = serde_json::json!({"name": 42});
|
||||||
|
let err = require_str(¶ms, "name").unwrap_err();
|
||||||
|
assert!(err.to_string().contains("missing 'name'"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_require_param_present() {
|
||||||
|
let params = serde_json::json!({"data": [1, 2, 3]});
|
||||||
|
assert_eq!(
|
||||||
|
require_param(¶ms, "data").unwrap(),
|
||||||
|
&serde_json::json!([1, 2, 3])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_require_param_missing() {
|
||||||
|
let params = serde_json::json!({});
|
||||||
|
let err = require_param(¶ms, "data").unwrap_err();
|
||||||
|
assert!(err.to_string().contains("missing 'data'"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -209,10 +209,10 @@ impl EndpointPattern {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check path prefix
|
// Check path prefix
|
||||||
if let Some(ref prefix) = self.path_prefix {
|
if let Some(ref prefix) = self.path_prefix
|
||||||
if !url_path.starts_with(prefix) {
|
&& !url_path.starts_with(prefix)
|
||||||
return false;
|
{
|
||||||
}
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check method
|
// Check method
|
||||||
@@ -237,13 +237,14 @@ impl EndpointPattern {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Support wildcard: *.example.com matches sub.example.com
|
// Support wildcard: *.example.com matches sub.example.com
|
||||||
if let Some(suffix) = self.host.strip_prefix("*.") {
|
if let Some(suffix) = self.host.strip_prefix("*.")
|
||||||
if url_host.ends_with(suffix) && url_host.len() > suffix.len() {
|
&& url_host.ends_with(suffix)
|
||||||
// Ensure there's a dot before the suffix (or it's the whole thing)
|
&& url_host.len() > suffix.len()
|
||||||
let prefix = &url_host[..url_host.len() - suffix.len()];
|
{
|
||||||
if prefix.ends_with('.') || prefix.is_empty() {
|
// Ensure there's a dot before the suffix (or it's the whole thing)
|
||||||
return true;
|
let prefix = &url_host[..url_host.len() - suffix.len()];
|
||||||
}
|
if prefix.ends_with('.') || prefix.is_empty() {
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,10 +292,10 @@ impl SecretsCapability {
|
|||||||
if pattern == name {
|
if pattern == name {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if let Some(prefix) = pattern.strip_suffix('*') {
|
if let Some(prefix) = pattern.strip_suffix('*')
|
||||||
if name.starts_with(prefix) {
|
&& name.starts_with(prefix)
|
||||||
return true;
|
{
|
||||||
}
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
false
|
false
|
||||||
|
|||||||
@@ -158,10 +158,10 @@ impl CredentialInjector {
|
|||||||
if pattern == name {
|
if pattern == name {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if let Some(prefix) = pattern.strip_suffix('*') {
|
if let Some(prefix) = pattern.strip_suffix('*')
|
||||||
if name.starts_with(prefix) {
|
&& name.starts_with(prefix)
|
||||||
return true;
|
{
|
||||||
}
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
false
|
false
|
||||||
@@ -169,7 +169,7 @@ impl CredentialInjector {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Inject a single credential into the result.
|
/// Inject a single credential into the result.
|
||||||
fn inject_credential(
|
pub(crate) fn inject_credential(
|
||||||
result: &mut InjectedCredentials,
|
result: &mut InjectedCredentials,
|
||||||
location: &CredentialLocation,
|
location: &CredentialLocation,
|
||||||
secret: &DecryptedSecret,
|
secret: &DecryptedSecret,
|
||||||
@@ -208,18 +208,19 @@ fn inject_credential(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a host matches a pattern (supports wildcards).
|
/// Check if a host matches a pattern (supports wildcards).
|
||||||
fn host_matches_pattern(host: &str, pattern: &str) -> bool {
|
pub(crate) fn host_matches_pattern(host: &str, pattern: &str) -> bool {
|
||||||
if pattern == host {
|
if pattern == host {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Support wildcard: *.example.com matches sub.example.com
|
// Support wildcard: *.example.com matches sub.example.com
|
||||||
if let Some(suffix) = pattern.strip_prefix("*.") {
|
if let Some(suffix) = pattern.strip_prefix("*.")
|
||||||
if host.ends_with(suffix) && host.len() > suffix.len() {
|
&& host.ends_with(suffix)
|
||||||
let prefix = &host[..host.len() - suffix.len()];
|
&& host.len() > suffix.len()
|
||||||
if prefix.ends_with('.') || prefix.is_empty() {
|
{
|
||||||
return true;
|
let prefix = &host[..host.len() - suffix.len()];
|
||||||
}
|
if prefix.ends_with('.') || prefix.is_empty() {
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+179
-7
@@ -39,10 +39,11 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
|
|
||||||
|
use crate::secrets::SecretsStore;
|
||||||
use crate::tools::registry::{ToolRegistry, WasmRegistrationError, WasmToolRegistration};
|
use crate::tools::registry::{ToolRegistry, WasmRegistrationError, WasmToolRegistration};
|
||||||
use crate::tools::wasm::capabilities_schema::CapabilitiesFile;
|
use crate::tools::wasm::capabilities_schema::CapabilitiesFile;
|
||||||
use crate::tools::wasm::{
|
use crate::tools::wasm::{
|
||||||
Capabilities, WasmError, WasmStorageError, WasmToolRuntime, WasmToolStore,
|
Capabilities, OAuthRefreshConfig, WasmError, WasmStorageError, WasmToolRuntime, WasmToolStore,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Error during WASM tool loading.
|
/// Error during WASM tool loading.
|
||||||
@@ -77,12 +78,23 @@ pub enum WasmLoadError {
|
|||||||
pub struct WasmToolLoader {
|
pub struct WasmToolLoader {
|
||||||
runtime: Arc<WasmToolRuntime>,
|
runtime: Arc<WasmToolRuntime>,
|
||||||
registry: Arc<ToolRegistry>,
|
registry: Arc<ToolRegistry>,
|
||||||
|
secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WasmToolLoader {
|
impl WasmToolLoader {
|
||||||
/// Create a new loader with the given runtime and registry.
|
/// Create a new loader with the given runtime and registry.
|
||||||
pub fn new(runtime: Arc<WasmToolRuntime>, registry: Arc<ToolRegistry>) -> Self {
|
pub fn new(runtime: Arc<WasmToolRuntime>, registry: Arc<ToolRegistry>) -> Self {
|
||||||
Self { runtime, registry }
|
Self {
|
||||||
|
runtime,
|
||||||
|
registry,
|
||||||
|
secrets_store: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the secrets store for credential injection in WASM tools.
|
||||||
|
pub fn with_secrets_store(mut self, store: Arc<dyn SecretsStore + Send + Sync>) -> Self {
|
||||||
|
self.secrets_store = Some(store);
|
||||||
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Load a single WASM tool from a file pair.
|
/// Load a single WASM tool from a file pair.
|
||||||
@@ -108,22 +120,24 @@ impl WasmToolLoader {
|
|||||||
}
|
}
|
||||||
let wasm_bytes = fs::read(wasm_path).await?;
|
let wasm_bytes = fs::read(wasm_path).await?;
|
||||||
|
|
||||||
// Read capabilities (optional)
|
// Read capabilities (optional) and extract OAuth refresh config
|
||||||
let capabilities = if let Some(cap_path) = capabilities_path {
|
let (capabilities, oauth_refresh) = if let Some(cap_path) = capabilities_path {
|
||||||
if cap_path.exists() {
|
if cap_path.exists() {
|
||||||
let cap_bytes = fs::read(cap_path).await?;
|
let cap_bytes = fs::read(cap_path).await?;
|
||||||
let cap_file = CapabilitiesFile::from_bytes(&cap_bytes)
|
let cap_file = CapabilitiesFile::from_bytes(&cap_bytes)
|
||||||
.map_err(|e| WasmLoadError::InvalidCapabilities(e.to_string()))?;
|
.map_err(|e| WasmLoadError::InvalidCapabilities(e.to_string()))?;
|
||||||
cap_file.to_capabilities()
|
let caps = cap_file.to_capabilities();
|
||||||
|
let oauth = resolve_oauth_refresh_config(&cap_file);
|
||||||
|
(caps, oauth)
|
||||||
} else {
|
} else {
|
||||||
tracing::warn!(
|
tracing::warn!(
|
||||||
path = %cap_path.display(),
|
path = %cap_path.display(),
|
||||||
"Capabilities file not found, using default (no permissions)"
|
"Capabilities file not found, using default (no permissions)"
|
||||||
);
|
);
|
||||||
Capabilities::default()
|
(Capabilities::default(), None)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Capabilities::default()
|
(Capabilities::default(), None)
|
||||||
};
|
};
|
||||||
|
|
||||||
// Register the tool
|
// Register the tool
|
||||||
@@ -136,6 +150,8 @@ impl WasmToolLoader {
|
|||||||
limits: None,
|
limits: None,
|
||||||
description: None,
|
description: None,
|
||||||
schema: None,
|
schema: None,
|
||||||
|
secrets_store: self.secrets_store.clone(),
|
||||||
|
oauth_refresh,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -293,6 +309,50 @@ impl WasmToolLoader {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Extract OAuth refresh configuration from a parsed capabilities file.
|
||||||
|
///
|
||||||
|
/// Returns `None` if there's no `auth.oauth` section or if the client_id
|
||||||
|
/// can't be resolved from any source (inline, env var, or built-in defaults).
|
||||||
|
///
|
||||||
|
/// Fallback chain for client_id:
|
||||||
|
/// `oauth.client_id` > env var (`oauth.client_id_env`) > `builtin_credentials()`
|
||||||
|
fn resolve_oauth_refresh_config(cap_file: &CapabilitiesFile) -> Option<OAuthRefreshConfig> {
|
||||||
|
let auth = cap_file.auth.as_ref()?;
|
||||||
|
let oauth = auth.oauth.as_ref()?;
|
||||||
|
|
||||||
|
let builtin = crate::cli::oauth_defaults::builtin_credentials(&auth.secret_name);
|
||||||
|
|
||||||
|
let client_id = oauth
|
||||||
|
.client_id
|
||||||
|
.clone()
|
||||||
|
.or_else(|| {
|
||||||
|
oauth
|
||||||
|
.client_id_env
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|env| std::env::var(env).ok())
|
||||||
|
})
|
||||||
|
.or_else(|| builtin.as_ref().map(|c| c.client_id.to_string()))?;
|
||||||
|
|
||||||
|
let client_secret = oauth
|
||||||
|
.client_secret
|
||||||
|
.clone()
|
||||||
|
.or_else(|| {
|
||||||
|
oauth
|
||||||
|
.client_secret_env
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|env| std::env::var(env).ok())
|
||||||
|
})
|
||||||
|
.or_else(|| builtin.as_ref().map(|c| c.client_secret.to_string()));
|
||||||
|
|
||||||
|
Some(OAuthRefreshConfig {
|
||||||
|
token_url: oauth.token_url.clone(),
|
||||||
|
client_id,
|
||||||
|
client_secret,
|
||||||
|
secret_name: auth.secret_name.clone(),
|
||||||
|
provider: auth.provider.clone(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Results from loading multiple tools.
|
/// Results from loading multiple tools.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct LoadResults {
|
pub struct LoadResults {
|
||||||
@@ -618,4 +678,116 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resolve_oauth_refresh_config_with_oauth() {
|
||||||
|
use crate::tools::wasm::capabilities_schema::{
|
||||||
|
AuthCapabilitySchema, CapabilitiesFile, OAuthConfigSchema,
|
||||||
|
};
|
||||||
|
|
||||||
|
let caps = CapabilitiesFile {
|
||||||
|
auth: Some(AuthCapabilitySchema {
|
||||||
|
secret_name: "google_oauth_token".to_string(),
|
||||||
|
provider: Some("google".to_string()),
|
||||||
|
oauth: Some(OAuthConfigSchema {
|
||||||
|
authorization_url: "https://accounts.google.com/o/oauth2/v2/auth".to_string(),
|
||||||
|
token_url: "https://oauth2.googleapis.com/token".to_string(),
|
||||||
|
client_id: Some("test-client-id".to_string()),
|
||||||
|
client_secret: Some("test-client-secret".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = super::resolve_oauth_refresh_config(&caps);
|
||||||
|
assert!(config.is_some());
|
||||||
|
|
||||||
|
let config = config.unwrap();
|
||||||
|
assert_eq!(config.token_url, "https://oauth2.googleapis.com/token");
|
||||||
|
assert_eq!(config.client_id, "test-client-id");
|
||||||
|
assert_eq!(config.client_secret, Some("test-client-secret".to_string()));
|
||||||
|
assert_eq!(config.secret_name, "google_oauth_token");
|
||||||
|
assert_eq!(config.provider, Some("google".to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resolve_oauth_refresh_config_no_auth() {
|
||||||
|
use crate::tools::wasm::capabilities_schema::CapabilitiesFile;
|
||||||
|
|
||||||
|
let caps = CapabilitiesFile::default();
|
||||||
|
let config = super::resolve_oauth_refresh_config(&caps);
|
||||||
|
assert!(config.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resolve_oauth_refresh_config_no_oauth() {
|
||||||
|
use crate::tools::wasm::capabilities_schema::{AuthCapabilitySchema, CapabilitiesFile};
|
||||||
|
|
||||||
|
let caps = CapabilitiesFile {
|
||||||
|
auth: Some(AuthCapabilitySchema {
|
||||||
|
secret_name: "manual_token".to_string(),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = super::resolve_oauth_refresh_config(&caps);
|
||||||
|
assert!(config.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resolve_oauth_refresh_config_no_client_id() {
|
||||||
|
use crate::tools::wasm::capabilities_schema::{
|
||||||
|
AuthCapabilitySchema, CapabilitiesFile, OAuthConfigSchema,
|
||||||
|
};
|
||||||
|
|
||||||
|
// A non-Google provider with no client_id anywhere should return None
|
||||||
|
let caps = CapabilitiesFile {
|
||||||
|
auth: Some(AuthCapabilitySchema {
|
||||||
|
secret_name: "unknown_provider_token".to_string(),
|
||||||
|
oauth: Some(OAuthConfigSchema {
|
||||||
|
authorization_url: "https://example.com/auth".to_string(),
|
||||||
|
token_url: "https://example.com/token".to_string(),
|
||||||
|
// No client_id, no client_id_env, no builtin
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = super::resolve_oauth_refresh_config(&caps);
|
||||||
|
assert!(config.is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_resolve_oauth_refresh_config_builtin_google() {
|
||||||
|
use crate::tools::wasm::capabilities_schema::{
|
||||||
|
AuthCapabilitySchema, CapabilitiesFile, OAuthConfigSchema,
|
||||||
|
};
|
||||||
|
|
||||||
|
// google_oauth_token should fall back to built-in credentials
|
||||||
|
let caps = CapabilitiesFile {
|
||||||
|
auth: Some(AuthCapabilitySchema {
|
||||||
|
secret_name: "google_oauth_token".to_string(),
|
||||||
|
provider: Some("google".to_string()),
|
||||||
|
oauth: Some(OAuthConfigSchema {
|
||||||
|
authorization_url: "https://accounts.google.com/o/oauth2/v2/auth".to_string(),
|
||||||
|
token_url: "https://oauth2.googleapis.com/token".to_string(),
|
||||||
|
// No inline client_id, should fall back to builtin
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let config = super::resolve_oauth_refresh_config(&caps);
|
||||||
|
assert!(config.is_some());
|
||||||
|
let config = config.unwrap();
|
||||||
|
assert!(!config.client_id.is_empty());
|
||||||
|
assert!(config.client_secret.is_some());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ pub use limits::{
|
|||||||
WasmResourceLimiter,
|
WasmResourceLimiter,
|
||||||
};
|
};
|
||||||
pub use runtime::{PreparedModule, WasmRuntimeConfig, WasmToolRuntime};
|
pub use runtime::{PreparedModule, WasmRuntimeConfig, WasmToolRuntime};
|
||||||
pub use wrapper::WasmToolWrapper;
|
pub use wrapper::{OAuthRefreshConfig, WasmToolWrapper};
|
||||||
|
|
||||||
// Capabilities (V2)
|
// Capabilities (V2)
|
||||||
pub use capabilities::{
|
pub use capabilities::{
|
||||||
@@ -108,10 +108,13 @@ pub use credential_injector::{CredentialInjector, InjectedCredentials, Injection
|
|||||||
pub use rate_limiter::{LimitType, RateLimitError, RateLimitResult, RateLimiter};
|
pub use rate_limiter::{LimitType, RateLimitError, RateLimitResult, RateLimiter};
|
||||||
|
|
||||||
// Storage (V2)
|
// Storage (V2)
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
pub use storage::LibSqlWasmToolStore;
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
|
pub use storage::PostgresWasmToolStore;
|
||||||
pub use storage::{
|
pub use storage::{
|
||||||
PostgresWasmToolStore, StoreToolParams, StoredCapabilities, StoredWasmTool,
|
StoreToolParams, StoredCapabilities, StoredWasmTool, StoredWasmToolWithBinary, ToolStatus,
|
||||||
StoredWasmToolWithBinary, ToolStatus, TrustLevel, WasmStorageError, WasmToolStore,
|
TrustLevel, WasmStorageError, WasmToolStore, compute_binary_hash, verify_binary_integrity,
|
||||||
compute_binary_hash, verify_binary_integrity,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Loader
|
// Loader
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ use std::collections::HashMap;
|
|||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::{DateTime, Utc};
|
use chrono::{DateTime, Utc};
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
use deadpool_postgres::Pool;
|
use deadpool_postgres::Pool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -263,16 +264,19 @@ pub fn verify_binary_integrity(binary: &[u8], expected_hash: &[u8]) -> bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// PostgreSQL implementation of WasmToolStore.
|
/// PostgreSQL implementation of WasmToolStore.
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
pub struct PostgresWasmToolStore {
|
pub struct PostgresWasmToolStore {
|
||||||
pool: Pool,
|
pool: Pool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
impl PostgresWasmToolStore {
|
impl PostgresWasmToolStore {
|
||||||
pub fn new(pool: Pool) -> Self {
|
pub fn new(pool: Pool) -> Self {
|
||||||
Self { pool }
|
Self { pool }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl WasmToolStore for PostgresWasmToolStore {
|
impl WasmToolStore for PostgresWasmToolStore {
|
||||||
async fn store(&self, params: StoreToolParams) -> Result<StoredWasmTool, WasmStorageError> {
|
async fn store(&self, params: StoreToolParams) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
@@ -538,6 +542,7 @@ impl WasmToolStore for PostgresWasmToolStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "postgres")]
|
||||||
fn row_to_tool(row: &tokio_postgres::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
fn row_to_tool(row: &tokio_postgres::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
let trust_level_str: String = row.get("trust_level");
|
let trust_level_str: String = row.get("trust_level");
|
||||||
let status_str: String = row.get("status");
|
let status_str: String = row.get("status");
|
||||||
@@ -559,6 +564,459 @@ fn row_to_tool(row: &tokio_postgres::Row) -> Result<StoredWasmTool, WasmStorageE
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ==================== libSQL implementation ====================
|
||||||
|
|
||||||
|
/// libSQL/Turso implementation of WasmToolStore.
|
||||||
|
///
|
||||||
|
/// Holds an `Arc<Database>` handle and creates a fresh connection per operation,
|
||||||
|
/// matching the connection-per-request pattern used by the main `LibSqlBackend`.
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
pub struct LibSqlWasmToolStore {
|
||||||
|
db: std::sync::Arc<libsql::Database>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
impl LibSqlWasmToolStore {
|
||||||
|
pub fn new(db: std::sync::Arc<libsql::Database>) -> Self {
|
||||||
|
Self { db }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn connect(&self) -> Result<libsql::Connection, WasmStorageError> {
|
||||||
|
self.db
|
||||||
|
.connect()
|
||||||
|
.map_err(|e| WasmStorageError::Database(format!("Connection failed: {}", e)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
#[async_trait]
|
||||||
|
impl WasmToolStore for LibSqlWasmToolStore {
|
||||||
|
async fn store(&self, params: StoreToolParams) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
|
let binary_hash = compute_binary_hash(¶ms.wasm_binary);
|
||||||
|
let id = Uuid::new_v4();
|
||||||
|
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||||
|
let schema_str = serde_json::to_string(¶ms.parameters_schema)
|
||||||
|
.map_err(|e| WasmStorageError::InvalidData(e.to_string()))?;
|
||||||
|
|
||||||
|
// Wrap INSERT + read-back in a transaction to prevent TOCTOU races
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let tx = conn
|
||||||
|
.transaction()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
tx.execute(
|
||||||
|
r#"
|
||||||
|
INSERT INTO wasm_tools (
|
||||||
|
id, user_id, name, version, description, wasm_binary, binary_hash,
|
||||||
|
parameters_schema, source_url, trust_level, status, created_at, updated_at
|
||||||
|
)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'active', ?11, ?11)
|
||||||
|
ON CONFLICT (user_id, name, version) DO UPDATE SET
|
||||||
|
description = excluded.description,
|
||||||
|
wasm_binary = excluded.wasm_binary,
|
||||||
|
binary_hash = excluded.binary_hash,
|
||||||
|
parameters_schema = excluded.parameters_schema,
|
||||||
|
source_url = excluded.source_url,
|
||||||
|
updated_at = ?11
|
||||||
|
"#,
|
||||||
|
libsql::params![
|
||||||
|
id.to_string(),
|
||||||
|
params.user_id.as_str(),
|
||||||
|
params.name.as_str(),
|
||||||
|
params.version.as_str(),
|
||||||
|
params.description.as_str(),
|
||||||
|
libsql::Value::Blob(params.wasm_binary),
|
||||||
|
libsql::Value::Blob(binary_hash),
|
||||||
|
schema_str.as_str(),
|
||||||
|
libsql_wasm_opt_text(params.source_url.as_deref()),
|
||||||
|
params.trust_level.to_string(),
|
||||||
|
now.as_str(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
// Read back the row within the same transaction
|
||||||
|
let mut rows = tx
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, version, description, parameters_schema,
|
||||||
|
source_url, trust_level, status, created_at, updated_at
|
||||||
|
FROM wasm_tools
|
||||||
|
WHERE user_id = ?1 AND name = ?2
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1
|
||||||
|
"#,
|
||||||
|
libsql::params![params.user_id.as_str(), params.name.as_str()],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
let row = rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
WasmStorageError::Database("Insert succeeded but row not found".into())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let tool = libsql_row_to_tool(&row)?;
|
||||||
|
|
||||||
|
tx.commit()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(tool)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get(&self, user_id: &str, name: &str) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, version, description, parameters_schema,
|
||||||
|
source_url, trust_level, status, created_at, updated_at
|
||||||
|
FROM wasm_tools
|
||||||
|
WHERE user_id = ?1 AND name = ?2 AND status = 'active'
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1
|
||||||
|
"#,
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
match rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
Some(row) => {
|
||||||
|
let tool = libsql_row_to_tool(&row)?;
|
||||||
|
match tool.status {
|
||||||
|
ToolStatus::Active => Ok(tool),
|
||||||
|
ToolStatus::Disabled => Err(WasmStorageError::Disabled),
|
||||||
|
ToolStatus::Quarantined => Err(WasmStorageError::Quarantined),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => Err(WasmStorageError::NotFound(name.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_with_binary(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<StoredWasmToolWithBinary, WasmStorageError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, version, description, wasm_binary, binary_hash,
|
||||||
|
parameters_schema, source_url, trust_level, status, created_at, updated_at
|
||||||
|
FROM wasm_tools
|
||||||
|
WHERE user_id = ?1 AND name = ?2 AND status = 'active'
|
||||||
|
ORDER BY version DESC
|
||||||
|
LIMIT 1
|
||||||
|
"#,
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
match rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
Some(row) => {
|
||||||
|
let wasm_binary: Vec<u8> = row
|
||||||
|
.get(5)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let binary_hash: Vec<u8> = row
|
||||||
|
.get(6)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
if !verify_binary_integrity(&wasm_binary, &binary_hash) {
|
||||||
|
tracing::error!(
|
||||||
|
user_id = user_id,
|
||||||
|
name = name,
|
||||||
|
"WASM binary integrity check failed"
|
||||||
|
);
|
||||||
|
return Err(WasmStorageError::IntegrityCheckFailed);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse metadata from the row (different column offsets due to binary/hash)
|
||||||
|
let tool = libsql_row_to_tool_with_offset(&row)?;
|
||||||
|
|
||||||
|
match tool.status {
|
||||||
|
ToolStatus::Active => Ok(StoredWasmToolWithBinary {
|
||||||
|
tool,
|
||||||
|
wasm_binary,
|
||||||
|
binary_hash,
|
||||||
|
}),
|
||||||
|
ToolStatus::Disabled => Err(WasmStorageError::Disabled),
|
||||||
|
ToolStatus::Quarantined => Err(WasmStorageError::Quarantined),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => Err(WasmStorageError::NotFound(name.to_string())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn get_capabilities(
|
||||||
|
&self,
|
||||||
|
tool_id: Uuid,
|
||||||
|
) -> Result<Option<StoredCapabilities>, WasmStorageError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, wasm_tool_id, http_allowlist, allowed_secrets, tool_aliases,
|
||||||
|
requests_per_minute, requests_per_hour, max_request_body_bytes,
|
||||||
|
max_response_body_bytes, workspace_read_prefixes, http_timeout_secs
|
||||||
|
FROM tool_capabilities
|
||||||
|
WHERE wasm_tool_id = ?1
|
||||||
|
"#,
|
||||||
|
libsql::params![tool_id.to_string()],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
match rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
Some(row) => {
|
||||||
|
let id_str: String = row
|
||||||
|
.get(0)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let tool_id_str: String = row
|
||||||
|
.get(1)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let http_allowlist_str: String = row.get::<String>(2).unwrap_or_default();
|
||||||
|
let allowed_secrets_str: String = row.get::<String>(3).unwrap_or_default();
|
||||||
|
let tool_aliases_str: String = row.get::<String>(4).unwrap_or_default();
|
||||||
|
let rpm: i64 = row.get::<i64>(5).unwrap_or(60);
|
||||||
|
let rph: i64 = row.get::<i64>(6).unwrap_or(1000);
|
||||||
|
let max_req: i64 = row.get::<i64>(7).unwrap_or(1048576);
|
||||||
|
let max_resp: i64 = row.get::<i64>(8).unwrap_or(10485760);
|
||||||
|
let ws_prefixes_str: String = row.get::<String>(9).unwrap_or_default();
|
||||||
|
let timeout: i64 = row.get::<i64>(10).unwrap_or(30);
|
||||||
|
|
||||||
|
let http_allowlist: Vec<EndpointPattern> =
|
||||||
|
serde_json::from_str(&http_allowlist_str).unwrap_or_default();
|
||||||
|
let allowed_secrets: Vec<String> =
|
||||||
|
serde_json::from_str(&allowed_secrets_str).unwrap_or_default();
|
||||||
|
let tool_aliases: HashMap<String, String> =
|
||||||
|
serde_json::from_str(&tool_aliases_str).unwrap_or_default();
|
||||||
|
let workspace_read_prefixes: Vec<String> =
|
||||||
|
serde_json::from_str(&ws_prefixes_str).unwrap_or_default();
|
||||||
|
|
||||||
|
Ok(Some(StoredCapabilities {
|
||||||
|
id: id_str
|
||||||
|
.parse()
|
||||||
|
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||||
|
wasm_tool_id: tool_id_str
|
||||||
|
.parse()
|
||||||
|
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||||
|
http_allowlist,
|
||||||
|
allowed_secrets,
|
||||||
|
tool_aliases,
|
||||||
|
requests_per_minute: rpm as u32,
|
||||||
|
requests_per_hour: rph as u32,
|
||||||
|
max_request_body_bytes: max_req,
|
||||||
|
max_response_body_bytes: max_resp,
|
||||||
|
workspace_read_prefixes,
|
||||||
|
http_timeout_secs: timeout as i32,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(&self, user_id: &str) -> Result<Vec<StoredWasmTool>, WasmStorageError> {
|
||||||
|
// SQLite doesn't have DISTINCT ON, so we use a subquery to get latest version per name
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let mut rows = conn
|
||||||
|
.query(
|
||||||
|
r#"
|
||||||
|
SELECT id, user_id, name, version, description, parameters_schema,
|
||||||
|
source_url, trust_level, status, created_at, updated_at
|
||||||
|
FROM wasm_tools
|
||||||
|
WHERE user_id = ?1
|
||||||
|
AND rowid IN (
|
||||||
|
SELECT MAX(rowid)
|
||||||
|
FROM wasm_tools
|
||||||
|
WHERE user_id = ?1
|
||||||
|
GROUP BY name
|
||||||
|
)
|
||||||
|
ORDER BY name
|
||||||
|
"#,
|
||||||
|
libsql::params![user_id],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
let mut tools = Vec::new();
|
||||||
|
while let Some(row) = rows
|
||||||
|
.next()
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||||
|
{
|
||||||
|
tools.push(libsql_row_to_tool(&row)?);
|
||||||
|
}
|
||||||
|
Ok(tools)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn update_status(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
name: &str,
|
||||||
|
status: ToolStatus,
|
||||||
|
) -> Result<(), WasmStorageError> {
|
||||||
|
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||||
|
let conn = self.connect()?;
|
||||||
|
|
||||||
|
let result = conn
|
||||||
|
.execute(
|
||||||
|
"UPDATE wasm_tools SET status = ?1, updated_at = ?2 WHERE user_id = ?3 AND name = ?4",
|
||||||
|
libsql::params![status.to_string(), now.as_str(), user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
if result == 0 {
|
||||||
|
return Err(WasmStorageError::NotFound(name.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, user_id: &str, name: &str) -> Result<bool, WasmStorageError> {
|
||||||
|
let conn = self.connect()?;
|
||||||
|
let result = conn
|
||||||
|
.execute(
|
||||||
|
"DELETE FROM wasm_tools WHERE user_id = ?1 AND name = ?2",
|
||||||
|
libsql::params![user_id, name],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(result > 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_wasm_opt_text(s: Option<&str>) -> libsql::Value {
|
||||||
|
match s {
|
||||||
|
Some(s) => libsql::Value::Text(s.to_string()),
|
||||||
|
None => libsql::Value::Null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_wasm_parse_ts(s: &str) -> Result<DateTime<Utc>, WasmStorageError> {
|
||||||
|
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) {
|
||||||
|
return Ok(dt.with_timezone(&Utc));
|
||||||
|
}
|
||||||
|
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") {
|
||||||
|
return Ok(ndt.and_utc());
|
||||||
|
}
|
||||||
|
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
|
||||||
|
return Ok(ndt.and_utc());
|
||||||
|
}
|
||||||
|
Err(WasmStorageError::InvalidData(format!(
|
||||||
|
"unparseable timestamp: {:?}",
|
||||||
|
s
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a tool row with standard column order (no binary columns).
|
||||||
|
/// Columns: id(0), user_id(1), name(2), version(3), description(4),
|
||||||
|
/// parameters_schema(5), source_url(6), trust_level(7), status(8),
|
||||||
|
/// created_at(9), updated_at(10)
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_row_to_tool(row: &libsql::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
|
libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a tool row when binary columns are present (get_with_binary query).
|
||||||
|
/// Columns: id(0), user_id(1), name(2), version(3), description(4),
|
||||||
|
/// wasm_binary(5), binary_hash(6),
|
||||||
|
/// parameters_schema(7), source_url(8), trust_level(9), status(10),
|
||||||
|
/// created_at(11), updated_at(12)
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
fn libsql_row_to_tool_with_offset(row: &libsql::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
|
libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 7, 8, 9, 10, 11, 12)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "libsql")]
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn libsql_row_to_tool_at(
|
||||||
|
row: &libsql::Row,
|
||||||
|
id_idx: i32,
|
||||||
|
user_id_idx: i32,
|
||||||
|
name_idx: i32,
|
||||||
|
version_idx: i32,
|
||||||
|
description_idx: i32,
|
||||||
|
schema_idx: i32,
|
||||||
|
source_url_idx: i32,
|
||||||
|
trust_level_idx: i32,
|
||||||
|
status_idx: i32,
|
||||||
|
created_at_idx: i32,
|
||||||
|
updated_at_idx: i32,
|
||||||
|
) -> Result<StoredWasmTool, WasmStorageError> {
|
||||||
|
let id_str: String = row
|
||||||
|
.get(id_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let trust_level_str: String = row
|
||||||
|
.get(trust_level_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let status_str: String = row
|
||||||
|
.get(status_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let schema_str: String = row
|
||||||
|
.get(schema_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let created_at_str: String = row
|
||||||
|
.get(created_at_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
let updated_at_str: String = row
|
||||||
|
.get(updated_at_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||||
|
|
||||||
|
Ok(StoredWasmTool {
|
||||||
|
id: id_str
|
||||||
|
.parse()
|
||||||
|
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||||
|
user_id: row
|
||||||
|
.get(user_id_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||||
|
name: row
|
||||||
|
.get(name_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||||
|
version: row
|
||||||
|
.get(version_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||||
|
description: row
|
||||||
|
.get(description_idx)
|
||||||
|
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||||
|
parameters_schema: serde_json::from_str(&schema_str).unwrap_or_default(),
|
||||||
|
source_url: row
|
||||||
|
.get::<String>(source_url_idx)
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty()),
|
||||||
|
trust_level: trust_level_str
|
||||||
|
.parse()
|
||||||
|
.map_err(WasmStorageError::InvalidData)?,
|
||||||
|
status: status_str.parse().map_err(WasmStorageError::InvalidData)?,
|
||||||
|
created_at: libsql_wasm_parse_ts(&created_at_str)?,
|
||||||
|
updated_at: libsql_wasm_parse_ts(&updated_at_str)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use crate::tools::wasm::storage::{
|
use crate::tools::wasm::storage::{
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user